From be4fac32af37fd982a14e27779f1756df1518e03 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 2 Jun 2014 22:58:03 -0700 Subject: [PATCH 001/240] Extract the state changes from the ensure storage method Instead of having the ensure storage (which ensures the atom/s storage details exist) also change the state of the flow have the prepare method which calls into ensure storage do these state changes instead. Change-Id: Ic7cd13ab360e68fc33e94c67a237adeef7040159 --- taskflow/engines/action_engine/engine.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index 8a370516..1d8455a3 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -161,10 +161,7 @@ class ActionEngine(base.EngineBase): self.notifier.notify(state, details) def _ensure_storage(self): - # NOTE(harlowja): signal to the tasks that exist that we are about to - # resume, if they have a previous state, they will now transition to - # a resuming state (and then to suspended). - self._change_state(states.RESUMING) # does nothing in PENDING state + """Ensure all contained atoms exist in the storage unit.""" for node in self._compilation.execution_graph.nodes_iter(): version = misc.get_version_string(node) if isinstance(node, retry.Retry): @@ -173,7 +170,6 @@ class ActionEngine(base.EngineBase): self.storage.ensure_task(node.name, version, node.save_as) if node.inject: self.storage.inject_atom_args(node.name, node.inject) - self._change_state(states.SUSPENDED) # does nothing in PENDING state @lock_utils.locked def prepare(self): @@ -181,7 +177,12 @@ class ActionEngine(base.EngineBase): raise exc.InvalidState("Can not prepare an engine" " which has not been compiled") if not self._storage_ensured: + # Set our own state to resuming -> (ensure atoms exist + # in storage) -> suspended in the storage unit and notify any + # attached listeners of these changes. + self._change_state(states.RESUMING) self._ensure_storage() + self._change_state(states.SUSPENDED) self._storage_ensured = True # At this point we can check to ensure all dependencies are either # flow/task provided or storage provided, if there are still missing From 8dc6e4f0fdb88aace0b4b127fe22ef4b9ab12928 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 16 Jul 2014 14:27:44 -0700 Subject: [PATCH 002/240] Add a sample script that can be used to build a test environment It can be difficult to figure out what to install to get a test environment that can be used to test all the opportunistic tests, zookeeper and mysql (with the right user and password and test database). This script provides a way to do this on rhel/ubuntu based environments and can be used to examine how to do this for other environment types. I use it with nova boot (on rhel images) and it has worked out great as a way to rebuild a clean test environment as I need. Change-Id: I9aef7e5536aa38e951a6411a8e83097339a43955 --- tools/env_builder.sh | 121 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 tools/env_builder.sh diff --git a/tools/env_builder.sh b/tools/env_builder.sh new file mode 100644 index 00000000..b6171c7d --- /dev/null +++ b/tools/env_builder.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +# This sets up a developer testing environment that can be used with various +# openstack projects (mainly for taskflow, but for others it should work +# fine also). +# +# Some things to note: +# +# - The mysql server that is setup is *not* secured. +# - The zookeeper server that is setup is *not* secured. +# - The downloads from external services are *not* certificate verified. +# +# Overall it should only be used for testing/developer environments (it was +# tested on ubuntu 14.04 and rhel 6.x, for other distributions some tweaking +# may be required). + +set -e +set -u + +# If on a debian environment this will make apt-get *not* prompt for passwords. +export DEBIAN_FRONTEND=noninteractive + +# http://www.unixcl.com/2009/03/print-text-in-style-box-bash-scripting.html +Box () { + str="$@" + len=$((${#str}+4)) + for i in $(seq $len); do echo -n '*'; done; + echo; echo "* "$str" *"; + for i in $(seq $len); do echo -n '*'; done; + echo +} + +set +e +python_27=`which python2.7` +set -e + +build_dir=`mktemp -d` +echo "Created build directory $build_dir..." +cd $build_dir + +# Get python 2.7 installed (if it's not). +if [ -z "$python_27" ]; then + py_file="Python-2.7.7.tgz" + py_base_file=${py_file%.*} + py_url="https://www.python.org/ftp/python/2.7.7/$py_file" + + Box "Building python 2.7..." + wget $py_url -O "$build_dir/$py_file" --no-check-certificate -nv + tar -xf "$py_file" + cd $build_dir/$py_base_file + ./configure --disable-ipv6 -q + make --quiet + + Box "Installing python 2.7..." + make altinstall >/dev/null 2>&1 + python_27=/usr/local/bin/python2.7 +fi + +set +e +pip_27=`which pip2.7` +set -e +if [ -z "$pip_27" ]; then + Box "Installing pip..." + wget "https://bootstrap.pypa.io/get-pip.py" \ + -O "$build_dir/get-pip.py" --no-check-certificate -nv + $python_27 "$build_dir/get-pip.py" >/dev/null 2>&1 + pip_27=/usr/local/bin/pip2.7 +fi + +Box "Installing tox..." +$pip_27 install -q 'tox>=1.6.1,<1.7.0' + +Box "Installing system packages..." +if [ -f "/etc/redhat-release" ]; then + yum install -y -q mysql-devel postgresql-devel mysql-server + mysqld="mysqld" +elif [ -f "/etc/debian_version" ]; then + apt-get -y -qq install libmysqlclient-dev mysql-server postgresql + mysqld="mysql" +else + echo "Unknown distribution!!" + lsb_release -a + exit 1 +fi + +Box "Setting up mysql..." +service $mysqld restart +/usr/bin/mysql --user="root" --execute='CREATE DATABASE 'openstack_citest'' +cat << EOF > $build_dir/mysql.sql +CREATE USER 'openstack_citest'@'localhost' IDENTIFIED BY 'openstack_citest'; +CREATE USER 'openstack_citest' IDENTIFIED BY 'openstack_citest'; +GRANT ALL PRIVILEGES ON *.* TO 'openstack_citest'@'localhost'; +GRANT ALL PRIVILEGES ON *.* TO 'openstack_citest'; +FLUSH PRIVILEGES; +EOF +/usr/bin/mysql --user="root" < $build_dir/mysql.sql + +Box "Installing zookeeper..." +zk_file="cloudera-cdh-4-0.x86_64.rpm" +zk_url="http://archive.cloudera.com/cdh4/one-click-install/redhat/6/x86_64/$zk_file" +if [ -f "/etc/redhat-release" ]; then + wget $zk_url -O $build_dir/$zk_file --no-check-certificate -nv + yum -y -q --nogpgcheck localinstall $build_dir/$zk_file + yum -y -q install zookeeper-server java + service zookeeper-server stop + service zookeeper-server init --force + mkdir -pv /var/lib/zookeeper + python -c "import random; print random.randint(1, 16384)" > /var/lib/zookeeper/myid + zookeeperd="zookeeper-server" +elif [ -f "/etc/debian_version" ]; then + apt-get install -y -qq zookeeperd + zookeeperd="zookeeper" +else + echo "Unknown distribution!!" + lsb_release -a + exit 1 +fi + +Box "Starting zookeeper..." +service $zookeeperd restart +service $zookeeperd status From 83be7e17a13f1148cec596d9d7c3b260bf8a20aa Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 12 Aug 2014 10:35:42 -0700 Subject: [PATCH 003/240] LOG which requeue filter callback failed When a requeue filter fails being called instead of just logging the exception and traceback also log the callback index which failed and the string representation of the callback that failed so that when/if it is later examined to determine the root cause these pieces of output can be used to isolate the failing callback. Also includes a drive-by commit to update the add requeue filter callback documentation to explain what the callback will be provided and expected to return. Change-Id: Idf1ae63cfa706551ac0361e802f3b082783e0b9d --- taskflow/engines/worker_based/dispatcher.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/taskflow/engines/worker_based/dispatcher.py b/taskflow/engines/worker_based/dispatcher.py index 9ff8ac10..dc4819b3 100644 --- a/taskflow/engines/worker_based/dispatcher.py +++ b/taskflow/engines/worker_based/dispatcher.py @@ -36,7 +36,11 @@ class TypeDispatcher(object): The callback will be activated before the message has been acked and it can be used to instruct the dispatcher to requeue the message - instead of processing it. + instead of processing it. The callback, when called, will be provided + two positional parameters; the first being the message data and the + second being the message object. Using these provided parameters the + filter should return a truthy object if the message should be requeued + and a falsey object if it should not. """ assert six.callable(callback), "Callback must be callable" self._requeue_filters.append(callback) @@ -44,14 +48,14 @@ class TypeDispatcher(object): def _collect_requeue_votes(self, data, message): # Returns how many of the filters asked for the message to be requeued. requeue_votes = 0 - for f in self._requeue_filters: + for i, cb in enumerate(self._requeue_filters): try: - if f(data, message): + if cb(data, message): requeue_votes += 1 except Exception: - LOG.exception("Failed calling requeue filter to determine" - " if message %r should be requeued.", - message.delivery_tag) + LOG.exception("Failed calling requeue filter %s '%s' to" + " determine if message %r should be requeued.", + i + 1, cb, message.delivery_tag) return requeue_votes def _requeue_log_error(self, message, errors): From c491683650f0912b6bc613a5a819aaf4f32c4340 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 12 Aug 2014 12:12:56 -0700 Subject: [PATCH 004/240] Allow WBE request transition timeout to be dynamic To enable longer (or shorter) timeouts for a WBE submitted request to transition out of the (PENDING, WAITING) states allow the transition timeout that was previously set to 60 seconds to be provided as a WBE configuration option (it still defaults to the previously fixed 60 seconds when it is not provided). Fixes bug 1356002 Change-Id: Idf384217004a334df03e2fff9150309fdfe08005 --- taskflow/engines/worker_based/engine.py | 16 +++++++++++++--- taskflow/engines/worker_based/executor.py | 9 ++++++--- taskflow/tests/unit/worker_based/test_engine.py | 8 +++++--- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/taskflow/engines/worker_based/engine.py b/taskflow/engines/worker_based/engine.py index e92e73f8..0c1b8ead 100644 --- a/taskflow/engines/worker_based/engine.py +++ b/taskflow/engines/worker_based/engine.py @@ -16,6 +16,7 @@ from taskflow.engines.action_engine import engine from taskflow.engines.worker_based import executor +from taskflow.engines.worker_based import protocol as pr from taskflow import storage as t_storage @@ -30,8 +31,15 @@ class WorkerBasedActionEngine(engine.ActionEngine): :param topics: list of workers topics to communicate with (this will also be learned by listening to the notifications that workers emit). - :keyword transport: transport to be used (e.g. amqp, memory, etc.) - :keyword transport_options: transport specific options + :param transport: transport to be used (e.g. amqp, memory, etc.) + :param transport_options: transport specific options + :param transition_timeout: numeric value (or None for infinite) to wait + for submitted remote requests to transition out + of the (PENDING, WAITING) request states. When + expired the associated task the request was made + for will have its result become a + `RequestTimeout` exception instead of its + normally returned value (or raised exception). """ _storage_factory = t_storage.SingleThreadedStorage @@ -45,7 +53,9 @@ class WorkerBasedActionEngine(engine.ActionEngine): exchange=self._conf.get('exchange', 'default'), topics=self._conf.get('topics', []), transport=self._conf.get('transport'), - transport_options=self._conf.get('transport_options')) + transport_options=self._conf.get('transport_options'), + transition_timeout=self._conf.get('transition_timeout', + pr.REQUEST_TIMEOUT)) def __init__(self, flow, flow_detail, backend, conf, **kwargs): super(WorkerBasedActionEngine, self).__init__( diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index 2b82f01a..959583d1 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -70,10 +70,12 @@ class PeriodicWorker(object): class WorkerTaskExecutor(executor.TaskExecutorBase): """Executes tasks on remote workers.""" - def __init__(self, uuid, exchange, topics, **kwargs): + def __init__(self, uuid, exchange, topics, + transition_timeout=pr.REQUEST_TIMEOUT, **kwargs): self._uuid = uuid self._topics = topics self._requests_cache = cache.RequestsCache() + self._transition_timeout = transition_timeout self._workers_cache = cache.WorkersCache() self._workers_arrival = threading.Condition() handlers = { @@ -157,10 +159,11 @@ class WorkerTaskExecutor(executor.TaskExecutorBase): self._requests_cache.cleanup(self._handle_expired_request) def _submit_task(self, task, task_uuid, action, arguments, - progress_callback, timeout=pr.REQUEST_TIMEOUT, **kwargs): + progress_callback, **kwargs): """Submit task request to a worker.""" request = pr.Request(task, task_uuid, action, arguments, - progress_callback, timeout, **kwargs) + progress_callback, self._transition_timeout, + **kwargs) # Get task's topic and publish request if topic was found. topic = self._workers_cache.get_topic_by_task(request.task_cls) diff --git a/taskflow/tests/unit/worker_based/test_engine.py b/taskflow/tests/unit/worker_based/test_engine.py index c966be5a..531dfe5a 100644 --- a/taskflow/tests/unit/worker_based/test_engine.py +++ b/taskflow/tests/unit/worker_based/test_engine.py @@ -46,7 +46,8 @@ class TestWorkerBasedActionEngine(test.MockTestCase): exchange='default', topics=[], transport=None, - transport_options=None) + transport_options=None, + transition_timeout=mock.ANY) ] self.assertEqual(self.master_mock.mock_calls, expected_calls) @@ -55,7 +56,7 @@ class TestWorkerBasedActionEngine(test.MockTestCase): _, flow_detail = pu.temporary_flow_detail() config = {'url': self.broker_url, 'exchange': self.exchange, 'topics': self.topics, 'transport': 'memory', - 'transport_options': {}} + 'transport_options': {}, 'transition_timeout': 200} engine.WorkerBasedActionEngine( flow, flow_detail, None, config).compile() @@ -65,6 +66,7 @@ class TestWorkerBasedActionEngine(test.MockTestCase): exchange=self.exchange, topics=self.topics, transport='memory', - transport_options={}) + transport_options={}, + transition_timeout=200) ] self.assertEqual(self.master_mock.mock_calls, expected_calls) From 98be1df5d1a1a10551e323a0fc24c6367bf97318 Mon Sep 17 00:00:00 2001 From: Brian Jarrett Date: Sat, 2 Aug 2014 22:01:57 -0600 Subject: [PATCH 005/240] Remove db locks and use random db names for tests Removed database locks against the database openstack_citest and implemented random name generation for new databases created just for persistence testing. Once the locks were removed, using 'template1' as the initial db for postgres connections was problematic, because postgres refuses to create databases when there are multiple connections to 'template1'. Switched to using 'postgres' as an initial db to use. Changed _reset_database to _init_db since we are always working with a brand new database, and removed the 'drop database' before a 'create database. Added a _remove_db method to remove the database once testing was finished. Change-Id: Iaf1c101df9c268da48db7432bcbc0467f6486bcd Closes-Bug: 1327469 --- .../unit/persistence/test_sql_persistence.py | 150 ++++++++++-------- 1 file changed, 80 insertions(+), 70 deletions(-) diff --git a/taskflow/tests/unit/persistence/test_sql_persistence.py b/taskflow/tests/unit/persistence/test_sql_persistence.py index b48f84a8..2baaa373 100644 --- a/taskflow/tests/unit/persistence/test_sql_persistence.py +++ b/taskflow/tests/unit/persistence/test_sql_persistence.py @@ -16,8 +16,8 @@ import contextlib import os +import random import tempfile -import threading import testtools @@ -29,12 +29,14 @@ import testtools # There are also "opportunistic" tests for both mysql and postgresql in here, # which allows testing against all 3 databases (sqlite, mysql, postgres) in # a properly configured unit test environment. For the opportunistic testing -# you need to set up a db named 'openstack_citest' with user 'openstack_citest' -# and password 'openstack_citest' on localhost. +# you need to set up a db user 'openstack_citest' with password +# 'openstack_citest' that has the permissions to create databases on +# localhost. USER = "openstack_citest" PASSWD = "openstack_citest" -DATABASE = "openstack_citest" +DATABASE = "tftest_" + ''.join(random.choice('0123456789') + for _ in range(12)) try: from taskflow.persistence.backends import impl_sqlalchemy @@ -50,7 +52,6 @@ MYSQL_VARIANTS = ('mysqldb', 'pymysql') from taskflow.persistence import backends from taskflow import test from taskflow.tests.unit.persistence import base -from taskflow.utils import lock_utils def _get_connect_string(backend, user, passwd, database=None, variant=None): @@ -97,7 +98,7 @@ def _postgres_exists(): return False engine = None try: - db_uri = _get_connect_string('postgres', USER, PASSWD, 'template1') + db_uri = _get_connect_string('postgres', USER, PASSWD, 'postgres') engine = sa.create_engine(db_uri) with contextlib.closing(engine.connect()): return True @@ -137,29 +138,31 @@ class SqlitePersistenceTest(test.TestCase, base.PersistenceTestMixin): class BackendPersistenceTestMixin(base.PersistenceTestMixin): """Specifies a backend type and does required setup and teardown.""" - LOCK_NAME = None def _get_connection(self): return self.backend.get_connection() - def _reset_database(self): - """Resets the database, and returns the uri to that database. + def _init_db(self): + """Sets up the database, and returns the uri to that database.""" + raise NotImplementedError() - Called *only* after locking succeeds. - """ + def _remove_db(self): + """Cleans up by removing the database once the tests are done.""" raise NotImplementedError() def setUp(self): super(BackendPersistenceTestMixin, self).setUp() self.backend = None - self.big_lock.acquire() - self.addCleanup(self.big_lock.release) try: + self.db_uri = self._init_db() + # Since we are using random database names, we need to make sure + # and remove our random database when we are done testing. + self.addCleanup(self._remove_db) conf = { - 'connection': self._reset_database(), + 'connection': self.db_uri } except Exception as e: - self.skipTest("Failed to reset your database;" + self.skipTest("Failed to create temporary database;" " testing being skipped due to: %s" % (e)) try: self.backend = impl_sqlalchemy.SQLAlchemyBackend(conf) @@ -174,25 +177,11 @@ class BackendPersistenceTestMixin(base.PersistenceTestMixin): @testtools.skipIf(not SQLALCHEMY_AVAILABLE, 'sqlalchemy is not available') @testtools.skipIf(not _mysql_exists(), 'mysql is not available') class MysqlPersistenceTest(BackendPersistenceTestMixin, test.TestCase): - LOCK_NAME = 'mysql_persistence_test' def __init__(self, *args, **kwargs): test.TestCase.__init__(self, *args, **kwargs) - # We need to make sure that each test goes through a set of locks - # to ensure that multiple tests are not modifying the database, - # dropping it, creating it at the same time. To accomplish this we use - # a lock that ensures multiple parallel processes can't run at the - # same time as well as a in-process lock to ensure that multiple - # threads can't run at the same time. - lock_path = os.path.join(tempfile.gettempdir(), - 'taskflow-%s.lock' % (self.LOCK_NAME)) - locks = [ - lock_utils.InterProcessLock(lock_path), - threading.RLock(), - ] - self.big_lock = lock_utils.MultiLock(locks) - def _reset_database(self): + def _init_db(self): working_variant = None for variant in MYSQL_VARIANTS: engine = None @@ -201,7 +190,6 @@ class MysqlPersistenceTest(BackendPersistenceTestMixin, test.TestCase): variant=variant) engine = sa.create_engine(db_uri) with contextlib.closing(engine.connect()) as conn: - conn.execute("DROP DATABASE IF EXISTS %s" % DATABASE) conn.execute("CREATE DATABASE %s" % DATABASE) working_variant = variant except Exception: @@ -216,60 +204,82 @@ class MysqlPersistenceTest(BackendPersistenceTestMixin, test.TestCase): break if not working_variant: variants = ", ".join(MYSQL_VARIANTS) - self.skipTest("Failed to find a mysql variant" - " (tried %s) that works; mysql testing" - " being skipped" % (variants)) + raise Exception("Failed to initialize MySQL db." + " Tried these variants: %s; MySQL testing" + " being skipped" % (variants)) else: return _get_connect_string('mysql', USER, PASSWD, database=DATABASE, variant=working_variant) - -@testtools.skipIf(not SQLALCHEMY_AVAILABLE, 'sqlalchemy is not available') -@testtools.skipIf(not _postgres_exists(), 'postgres is not available') -class PostgresPersistenceTest(BackendPersistenceTestMixin, test.TestCase): - LOCK_NAME = 'postgres_persistence_test' - - def __init__(self, *args, **kwargs): - test.TestCase.__init__(self, *args, **kwargs) - # We need to make sure that each test goes through a set of locks - # to ensure that multiple tests are not modifying the database, - # dropping it, creating it at the same time. To accomplish this we use - # a lock that ensures multiple parallel processes can't run at the - # same time as well as a in-process lock to ensure that multiple - # threads can't run at the same time. - lock_path = os.path.join(tempfile.gettempdir(), - 'taskflow-%s.lock' % (self.LOCK_NAME)) - locks = [ - lock_utils.InterProcessLock(lock_path), - threading.RLock(), - ] - self.big_lock = lock_utils.MultiLock(locks) - - def _reset_database(self): + def _remove_db(self): engine = None try: - # Postgres can't operate on the database it's connected to, that's - # why we connect to the default template database 'template1' and - # then drop and create the desired database. - db_uri = _get_connect_string('postgres', USER, PASSWD, - database='template1') - engine = sa.create_engine(db_uri) + engine = sa.create_engine(self.db_uri) with contextlib.closing(engine.connect()) as conn: - conn.connection.set_isolation_level(0) conn.execute("DROP DATABASE IF EXISTS %s" % DATABASE) - conn.connection.set_isolation_level(1) - with contextlib.closing(engine.connect()) as conn: - conn.connection.set_isolation_level(0) - conn.execute("CREATE DATABASE %s" % DATABASE) - conn.connection.set_isolation_level(1) + except Exception as e: + raise Exception('Failed to remove temporary database: %s' % (e)) + finally: + if engine is not None: + try: + engine.dispose() + except Exception: + pass + + +@testtools.skipIf(not SQLALCHEMY_AVAILABLE, 'sqlalchemy is not available') +@testtools.skipIf(not _postgres_exists(), 'postgres is not available') +class PostgresPersistenceTest(BackendPersistenceTestMixin, test.TestCase): + + def __init__(self, *args, **kwargs): + test.TestCase.__init__(self, *args, **kwargs) + + def _init_db(self): + engine = None + try: + # Postgres can't operate on the database it's connected to, that's + # why we connect to the database 'postgres' and then create the + # desired database. + db_uri = _get_connect_string('postgres', USER, PASSWD, + database='postgres') + engine = sa.create_engine(db_uri) + with contextlib.closing(engine.connect()) as conn: + conn.connection.set_isolation_level(0) + conn.execute("CREATE DATABASE %s" % DATABASE) + conn.connection.set_isolation_level(1) + except Exception as e: + raise Exception('Failed to initialize PostgreSQL db: %s' % (e)) + finally: + if engine is not None: + try: + engine.dispose() + except Exception: + pass + return _get_connect_string('postgres', USER, PASSWD, + database=DATABASE) + + def _remove_db(self): + engine = None + try: + # Postgres can't operate on the database it's connected to, that's + # why we connect to the 'postgres' database and then drop the + # database. + db_uri = _get_connect_string('postgres', USER, PASSWD, + database='postgres') + engine = sa.create_engine(db_uri) + with contextlib.closing(engine.connect()) as conn: + conn.connection.set_isolation_level(0) + conn.execute("DROP DATABASE IF EXISTS %s" % DATABASE) + conn.connection.set_isolation_level(1) + except Exception as e: + raise Exception('Failed to remove temporary database: %s' % (e)) finally: if engine is not None: try: engine.dispose() except Exception: pass - return _get_connect_string('postgres', USER, PASSWD, database=DATABASE) @testtools.skipIf(not SQLALCHEMY_AVAILABLE, 'sqlalchemy is not available') From b4738fd34ed0eb185d728dcf128aad78a9e150e3 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 12 Aug 2014 08:22:11 -0700 Subject: [PATCH 006/240] Expand documention on failures and wrapped failures types To make it more clear why these objects and types exist and when and why they are used and reraised increase the verbosity of the comments and associated inline documentation on these two classes to help explain there existence & usefulness. Change-Id: I742bbcd69c71df80e6de7becc8aee153bf16b34b --- taskflow/exceptions.py | 15 ++++++++----- taskflow/utils/misc.py | 51 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/taskflow/exceptions.py b/taskflow/exceptions.py index 55c889ca..f758e924 100644 --- a/taskflow/exceptions.py +++ b/taskflow/exceptions.py @@ -132,12 +132,17 @@ class MultipleChoices(TaskFlowException): # Others. class WrappedFailure(Exception): - """Wraps one or several failures. + """Wraps one or several failure objects. - When exception cannot be re-raised (for example, because - the value and traceback is lost in serialization) or - there are several exceptions, we wrap corresponding Failure - objects into this exception class. + When exception/s cannot be re-raised (for example, because the value and + traceback are lost in serialization) or there are several exceptions active + at the same time (due to more than one thread raising exceptions), we will + wrap the corresponding failure objects into this exception class and + *may* reraise this exception type to allow users to handle the contained + failures/causes as they see fit... + + See the failure class documentation for a more comprehensive set of reasons + why this object *may* be reraised instead of the original exception. """ def __init__(self, causes): diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 65f21e24..37e70362 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -546,7 +546,7 @@ def are_equal_exc_info_tuples(ei1, ei2): @contextlib.contextmanager def capture_failure(): - """Captures the occuring exception and provides a failure back. + """Captures the occurring exception and provides a failure object back. This will save the current exception information and yield back a failure object for the caller to use (it will raise a runtime error if @@ -579,8 +579,53 @@ def capture_failure(): class Failure(object): """Object that represents failure. - Failure objects encapsulate exception information so that - it can be re-used later to re-raise or inspect. + Failure objects encapsulate exception information so that they can be + re-used later to re-raise, inspect, examine, log, print, serialize, + deserialize... + + One example where they are dependened upon is in the WBE engine. When a + remote worker throws an exception, the WBE based engine will receive that + exception and desire to reraise it to the user/caller of the WBE based + engine for appropriate handling (this matches the behavior of non-remote + engines). To accomplish this a failure object (or a to_dict() form) would + be sent over the WBE channel and the WBE based engine would deserialize it + and use this objects reraise() method to cause an exception that contains + similar/equivalent information as the original exception to be reraised, + allowing the user (or the WBE engine itself) to then handle the worker + failure/exception as they desire. + + For those who are curious, here are a few reasons why the original + exception itself *may* not be reraised and instead a reraised wrapped + failure exception object will be instead. These explanations are *only* + applicable when a failure object is serialized and deserialized (when it is + retained inside the python process that the exception was created in the + the original exception can be reraised correctly without issue). + + * Traceback objects are not serializable/recreatable, since they contain + references to stack frames at the location where the exception was + raised. When a failure object is serialized and sent across a channel + and recreated it is *not* possible to restore the original traceback and + originating stack frames. + * The original exception *type* can not be guaranteed to be found, workers + can run code that is not accessible/available when the failure is being + deserialized. Even if it was possible to use pickle safely it would not + be possible to find the originating exception or associated code in this + situation. + * The original exception *type* can not be guaranteed to be constructed in + a *correct* manner. At the time of failure object creation the exception + has already been created and the failure object can not assume it has + knowledge (or the ability) to recreate the original type of the captured + exception (this is especially hard if the original exception was created + via a complex process via some custom exception constructor). + * The original exception *type* can not be guaranteed to be constructed in + a *safe* manner. Importing *foreign* exception types dynamically can be + problematic when not done correctly and in a safe manner; since failure + objects can capture any exception it would be *unsafe* to try to import + those exception types namespaces and modules on the receiver side + dynamically (this would create similar issues as the ``pickle`` module in + python has where foreign modules can be imported, causing those modules + to have code ran when this happens, and this can cause issues and + side-effects that the receiver would not have intended to have caused). """ DICT_VERSION = 1 From 296e660cd3036db46a58ed3b16c907c769454f05 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 30 Jun 2014 20:26:26 -0700 Subject: [PATCH 007/240] Have the dispatch_job function return a future To make it easier to add in a multi-threaded conductor convert the base dispatch_job function to return a future object. This future object will contain a single result, whether the job should be consumed or abandoned. In the single threaded conductor its dispatch_job function will return a future, after completing the job (in a multi threaded conductor it would not return a future after doing the work). Change-Id: I077334820d36c64e272e93d158e3a0cd0d66a937 --- taskflow/conductors/base.py | 7 ++++--- taskflow/conductors/single_threaded.py | 6 ++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/taskflow/conductors/base.py b/taskflow/conductors/base.py index e7c9887a..c881346e 100644 --- a/taskflow/conductors/base.py +++ b/taskflow/conductors/base.py @@ -108,9 +108,10 @@ class Conductor(object): """Dispatches a claimed job for work completion. Accepts a single (already claimed) job and causes it to be run in - an engine. Returns a boolean that signifies whether the job should - be consumed. The job is consumed upon completion (unless False is - returned which will signify the job should be abandoned instead). + an engine. Returns a future object that represented the work to be + completed sometime in the future. The future should return a single + boolean from its result() method. This boolean determines whether the + job will be consumed (true) or whether it should be abandoned (false). :param job: A job instance that has already been claimed by the jobboard. diff --git a/taskflow/conductors/single_threaded.py b/taskflow/conductors/single_threaded.py index 5e78e348..23994e79 100644 --- a/taskflow/conductors/single_threaded.py +++ b/taskflow/conductors/single_threaded.py @@ -21,6 +21,7 @@ from taskflow.conductors import base from taskflow import exceptions as excp from taskflow.listeners import logging as logging_listener from taskflow.types import timing as tt +from taskflow.utils import async_utils from taskflow.utils import lock_utils LOG = logging.getLogger(__name__) @@ -116,7 +117,7 @@ class SingleThreadedConductor(base.Conductor): job, exc_info=True) else: LOG.info("Job completed successfully: %s", job) - return consume + return async_utils.make_completed_future(consume) def run(self): self._dead.clear() @@ -136,12 +137,13 @@ class SingleThreadedConductor(base.Conductor): continue consume = False try: - consume = self._dispatch_job(job) + f = self._dispatch_job(job) except Exception: LOG.warn("Job dispatching failed: %s", job, exc_info=True) else: dispatched += 1 + consume = f.result() try: if consume: self._jobboard.consume(job, self._name) From e389ee61d109a6b1e4908bde97547d5dbda596e3 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 17 Aug 2014 00:22:12 -0700 Subject: [PATCH 008/240] Reject WBE messages if they can't be put in an ack state Attempt to reject messages that could not be acknowledged so that those messages could be resent (requeueing is not appropriate since it will just cause the message to come back). Fixes bug 1364543 Change-Id: I2cb33b39be950528fa9e22d9a11722bedd0927aa --- taskflow/engines/worker_based/dispatcher.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/taskflow/engines/worker_based/dispatcher.py b/taskflow/engines/worker_based/dispatcher.py index 9ff8ac10..9321b99b 100644 --- a/taskflow/engines/worker_based/dispatcher.py +++ b/taskflow/engines/worker_based/dispatcher.py @@ -93,6 +93,9 @@ class TypeDispatcher(object): LOG.debug("AMQP message %r acknowledged.", message.delivery_tag) handler(data, message) + else: + message.reject_log_error(logger=LOG, + errors=(kombu_exc.MessageStateError,)) def on_message(self, data, message): """This method is called on incoming messages.""" From a96f49b9a5c73985187e75ba4eba6b946875c92a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 5 Sep 2014 11:50:22 -0700 Subject: [PATCH 009/240] Ensure the cachedproperty creation/setting is thread-safe When the cachedproperty descriptor is attached to an object that needs to be only created/set by one thread at a time we should ensure that this is done safely by using a lock to prevent multiple threads from creating and assigning the associated attribute. Fixes bug 1366156 Change-Id: I0545683f83402097f54c34a6b737904e6edd85b3 --- taskflow/tests/unit/test_utils.py | 33 +++++++++++++++++++++++++++++++ taskflow/utils/misc.py | 21 ++++++++++++++------ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/taskflow/tests/unit/test_utils.py b/taskflow/tests/unit/test_utils.py index 4518f961..5d4b7615 100644 --- a/taskflow/tests/unit/test_utils.py +++ b/taskflow/tests/unit/test_utils.py @@ -17,7 +17,10 @@ import collections import functools import inspect +import random import sys +import threading +import time import six import testtools @@ -437,6 +440,36 @@ class CachedPropertyTest(test.TestCase): self.assertEqual(None, inspect.getdoc(A.b)) + def test_threaded_access_property(self): + called = collections.deque() + + class A(object): + @misc.cachedproperty + def b(self): + called.append(1) + # NOTE(harlowja): wait for a little and give some time for + # another thread to potentially also get in this method to + # also create the same property... + time.sleep(random.random() * 0.5) + return 'b' + + a = A() + threads = [] + try: + for _i in range(0, 20): + t = threading.Thread(target=lambda: a.b) + t.daemon = True + threads.append(t) + for t in threads: + t.start() + finally: + while threads: + t = threads.pop() + t.join() + + self.assertEqual(1, len(called)) + self.assertEqual('b', a.b) + class AttrDictTest(test.TestCase): def test_ok_create(self): diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 8e5e1921..a993f296 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -27,6 +27,7 @@ import os import re import string import sys +import threading import time import traceback @@ -163,7 +164,7 @@ def decode_json(raw_data, root_types=(dict,)): class cachedproperty(object): - """A descriptor property that is only evaluated once.. + """A *thread-safe* descriptor property that is only evaluated once. This caching descriptor can be placed on instance methods to translate those methods into properties that will be cached in the instance (avoiding @@ -176,6 +177,7 @@ class cachedproperty(object): after the first call to 'get_thing' occurs. """ def __init__(self, fget): + self._lock = threading.RLock() # If a name is provided (as an argument) then this will be the string # to place the cached attribute under if not then it will be the # function itself to be wrapped into a property. @@ -205,12 +207,19 @@ class cachedproperty(object): def __get__(self, instance, owner): if instance is None: return self - try: + # Quick check to see if this already has been made (before acquiring + # the lock). This is safe to do since we don't allow deletion after + # being created. + if hasattr(instance, self._attr_name): return getattr(instance, self._attr_name) - except AttributeError: - value = self._fget(instance) - setattr(instance, self._attr_name, value) - return value + else: + with self._lock: + try: + return getattr(instance, self._attr_name) + except AttributeError: + value = self._fget(instance) + setattr(instance, self._attr_name, value) + return value def wallclock(): From 5e1fe405a21997d7f3c75702dcacc1bcf80d5bac Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 23 Aug 2014 18:36:39 -0700 Subject: [PATCH 010/240] Raise a runtime error when mixed green/non-green futures To avoid the dead-lock scenario when green futures are being waited on in the same call as non-green futures just disallow this rare and unlikely scenario in the first place. Change-Id: I5b57f7aae94ae40d22047b0b085e4d99e034c2a6 --- taskflow/utils/async_utils.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/taskflow/utils/async_utils.py b/taskflow/utils/async_utils.py index 0599870d..6d280dfe 100644 --- a/taskflow/utils/async_utils.py +++ b/taskflow/utils/async_utils.py @@ -22,16 +22,25 @@ from taskflow.utils import eventlet_utils as eu def wait_for_any(fs, timeout=None): """Wait for one of the futures to complete. - Works correctly with both green and non-green futures. + Works correctly with both green and non-green futures (but not both + together, since this can't be guaranteed to avoid dead-lock due to how + the waiting implementations are different when green threads are being + used). - Returns pair (done, not_done). + Returns pair (done futures, not done futures). """ - any_green = any(isinstance(f, eu.GreenFuture) for f in fs) - if any_green: - return eu.wait_for_any(fs, timeout=timeout) - else: + green_fs = sum(1 for f in fs if isinstance(f, eu.GreenFuture)) + if not green_fs: return tuple(futures.wait(fs, timeout=timeout, return_when=futures.FIRST_COMPLETED)) + else: + non_green_fs = len(fs) - green_fs + if non_green_fs: + raise RuntimeError("Can not wait on %s green futures and %s" + " non-green futures in the same `wait_for_any`" + " call" % (green_fs, non_green_fs)) + else: + return eu.wait_for_any(fs, timeout=timeout) def make_completed_future(result): From e68d72f66e387f5b9f9a543d8ac3000ef5cfbbdc Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 18 Jul 2014 12:22:00 -0700 Subject: [PATCH 011/240] Be smarter about required flow symbols Instead of blindly assuming all the symbols that are provided automatically work for all flows even if the flow has ordering constraints we should set the base flow class requires property to be abstract and provide flow specific properties that can do the appropriate analysis to determine what the flows unsatisfied symbol requirements actually are. Part of blueprint taskflow-improved-scoping Change-Id: Ie149c05b3305c5bfff9d9f2c05e7e064c3a6d0c7 --- doc/source/inputs_and_outputs.rst | 2 +- taskflow/flow.py | 34 ++++++++--------------------- taskflow/patterns/graph_flow.py | 25 +++++++++++++++++---- taskflow/patterns/linear_flow.py | 15 +++++++++++-- taskflow/patterns/unordered_flow.py | 12 ++++++++++ taskflow/types/graph.py | 22 +++++++++++++++++++ 6 files changed, 78 insertions(+), 32 deletions(-) diff --git a/doc/source/inputs_and_outputs.rst b/doc/source/inputs_and_outputs.rst index 34fb1bad..5ecf65e5 100644 --- a/doc/source/inputs_and_outputs.rst +++ b/doc/source/inputs_and_outputs.rst @@ -49,7 +49,7 @@ For example: ... MyTask(requires='b', provides='d') ... ) >>> flow.requires - set(['a']) + frozenset(['a']) >>> sorted(flow.provides) ['b', 'c', 'd'] diff --git a/taskflow/flow.py b/taskflow/flow.py index 5533ed4e..0359125d 100644 --- a/taskflow/flow.py +++ b/taskflow/flow.py @@ -34,10 +34,7 @@ class Flow(object): NOTE(harlowja): if a flow is placed in another flow as a subflow, a desired way to compose flows together, then it is valid and permissible that during - execution the subflow & parent flow may be flattened into a new flow. Since - a flow is just a 'structuring' concept this is typically a behavior that - should not be worried about (as it is not visible to the user), but it is - worth mentioning here. + compilation the subflow & parent flow *may* be flattened into a new flow. """ def __init__(self, name, retry=None): @@ -45,7 +42,7 @@ class Flow(object): self._retry = retry # NOTE(akarpinska): if retry doesn't have a name, # the name of its owner will be assigned - if self._retry and self._retry.name is None: + if self._retry is not None and self._retry.name is None: self._retry.name = self.name + "_retry" @property @@ -93,27 +90,14 @@ class Flow(object): @property def provides(self): - """Set of result names provided by the flow. - - Includes names of all the outputs provided by atoms of this flow. - """ + """Set of symbol names provided by the flow.""" provides = set() - if self._retry: + if self._retry is not None: provides.update(self._retry.provides) - for subflow in self: - provides.update(subflow.provides) - return provides + for item in self: + provides.update(item.provides) + return frozenset(provides) - @property + @abc.abstractproperty def requires(self): - """Set of argument names required by the flow. - - Includes names of all the inputs required by atoms of this - flow, but not provided within the flow itself. - """ - requires = set() - if self._retry: - requires.update(self._retry.requires) - for subflow in self: - requires.update(subflow.requires) - return requires - self.provides + """Set of *unsatisfied* symbol names required by the flow.""" diff --git a/taskflow/patterns/graph_flow.py b/taskflow/patterns/graph_flow.py index 7db4fee2..9717b211 100644 --- a/taskflow/patterns/graph_flow.py +++ b/taskflow/patterns/graph_flow.py @@ -16,8 +16,6 @@ import collections -from networkx.algorithms import traversal - from taskflow import exceptions as exc from taskflow import flow from taskflow.types import graph as gr @@ -170,6 +168,26 @@ class Flow(flow.Flow): for (u, v, e_data) in self._get_subgraph().edges_iter(data=True): yield (u, v, e_data) + @property + def requires(self): + requires = set() + retry_provides = set() + if self._retry is not None: + requires.update(self._retry.requires) + retry_provides.update(self._retry.provides) + g = self._get_subgraph() + for item in g.nodes_iter(): + item_requires = item.requires - retry_provides + # Now scan predecessors to see if they provide what we want. + if item_requires: + for pred_item in g.bfs_predecessors_iter(item): + item_requires = item_requires - pred_item.provides + if not item_requires: + break + if item_requires: + requires.update(item_requires) + return frozenset(requires) + class TargetedFlow(Flow): """Graph flow with a target. @@ -223,8 +241,7 @@ class TargetedFlow(Flow): if self._target is None: return self._graph nodes = [self._target] - nodes.extend(dst for _src, dst in - traversal.dfs_edges(self._graph.reverse(), self._target)) + nodes.extend(self._graph.bfs_predecessors_iter(self._target)) self._subgraph = self._graph.subgraph(nodes) self._subgraph.freeze() return self._subgraph diff --git a/taskflow/patterns/linear_flow.py b/taskflow/patterns/linear_flow.py index 48b4d3cb..14799b82 100644 --- a/taskflow/patterns/linear_flow.py +++ b/taskflow/patterns/linear_flow.py @@ -74,7 +74,18 @@ class Flow(flow.Flow): for child in self._children: yield child + @property + def requires(self): + requires = set() + prior_provides = set() + if self._retry is not None: + requires.update(self._retry.requires) + prior_provides.update(self._retry.provides) + for item in self: + requires.update(item.requires - prior_provides) + prior_provides.update(item.provides) + return frozenset(requires) + def iter_links(self): - for src, dst in zip(self._children[:-1], - self._children[1:]): + for src, dst in zip(self._children[:-1], self._children[1:]): yield (src, dst, _LINK_METADATA.copy()) diff --git a/taskflow/patterns/unordered_flow.py b/taskflow/patterns/unordered_flow.py index a8377960..b32fbc09 100644 --- a/taskflow/patterns/unordered_flow.py +++ b/taskflow/patterns/unordered_flow.py @@ -92,3 +92,15 @@ class Flow(flow.Flow): # NOTE(imelnikov): children in unordered flow have no dependencies # between each other due to invariants retained during construction. return iter(()) + + @property + def requires(self): + requires = set() + retry_provides = set() + if self._retry is not None: + requires.update(self._retry.requires) + retry_provides.update(self._retry.provides) + for item in self: + item_requires = item.requires - retry_provides + requires.update(item_requires) + return frozenset(requires) diff --git a/taskflow/types/graph.py b/taskflow/types/graph.py index d3e2bae2..22b6b71c 100644 --- a/taskflow/types/graph.py +++ b/taskflow/types/graph.py @@ -14,6 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. +import collections + import networkx as nx import six @@ -98,6 +100,26 @@ class DiGraph(nx.DiGraph): if not len(self.predecessors(n)): yield n + def bfs_predecessors_iter(self, n): + """Iterates breadth first over *all* predecessors of a given node. + + This will go through the nodes predecessors, then the predecessor nodes + predecessors and so on until no more predecessors are found. + + NOTE(harlowja): predecessor cycles (if they exist) will not be iterated + over more than once (this prevents infinite iteration). + """ + visited = set([n]) + queue = collections.deque(self.predecessors_iter(n)) + while queue: + pred = queue.popleft() + if pred not in visited: + yield pred + visited.add(pred) + for pred_pred in self.predecessors_iter(pred): + if pred_pred not in visited: + queue.append(pred_pred) + def merge_graphs(graphs, allow_overlaps=False): """Merges a bunch of graphs into a single graph.""" From fa077c953fac48cf8fcb8ef4d178017b93d4ffce Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 21 Jul 2014 18:25:49 -0700 Subject: [PATCH 012/240] Revamp the symbol lookup mechanism To complement the future changes in patterns we also want to allow the execution of patterns to be affected in a similar manner so that symbol lookup is no longer as confined as it was. This change adds in the following: - Symbol lookup by walking through an atoms contained scope/s. - Better error messaging when symbols are not found. - Adjusted & new tests (existing ones work). - Better logging of the symbol lookup mechanism (helpful during debugging, although it is very verbose...) Part of blueprint taskflow-improved-scoping Change-Id: Id921a4abd9bf2b7b5c5a762337f8e90e8f1fe194 --- taskflow/engines/action_engine/compiler.py | 175 ++++++----- taskflow/engines/action_engine/engine.py | 4 +- .../engines/action_engine/retry_action.py | 7 +- taskflow/engines/action_engine/runtime.py | 17 +- taskflow/engines/action_engine/scopes.py | 119 ++++++++ taskflow/engines/action_engine/task_action.py | 11 +- taskflow/patterns/graph_flow.py | 2 +- taskflow/storage.py | 272 ++++++++++++++---- .../tests/unit/action_engine/test_compile.py | 62 ++-- .../tests/unit/action_engine/test_runner.py | 2 +- .../tests/unit/test_action_engine_scoping.py | 248 ++++++++++++++++ taskflow/tests/unit/test_storage.py | 17 -- taskflow/types/tree.py | 5 + taskflow/utils/misc.py | 22 +- 14 files changed, 735 insertions(+), 228 deletions(-) create mode 100644 taskflow/engines/action_engine/scopes.py create mode 100644 taskflow/tests/unit/test_action_engine_scoping.py diff --git a/taskflow/engines/action_engine/compiler.py b/taskflow/engines/action_engine/compiler.py index 32cb58c8..41fe5f96 100644 --- a/taskflow/engines/action_engine/compiler.py +++ b/taskflow/engines/action_engine/compiler.py @@ -15,53 +15,34 @@ # under the License. import logging +import threading from taskflow import exceptions as exc from taskflow import flow from taskflow import retry from taskflow import task from taskflow.types import graph as gr +from taskflow.types import tree as tr +from taskflow.utils import lock_utils from taskflow.utils import misc LOG = logging.getLogger(__name__) class Compilation(object): - """The result of a compilers compile() is this *immutable* object. + """The result of a compilers compile() is this *immutable* object.""" - For now it is just a execution graph but in the future it will grow to - include more methods & properties that help the various runtime units - execute in a more optimal & featureful manner. - """ - def __init__(self, execution_graph): + def __init__(self, execution_graph, hierarchy): self._execution_graph = execution_graph + self._hierarchy = hierarchy @property def execution_graph(self): return self._execution_graph - -class PatternCompiler(object): - """Compiles patterns & atoms into a compilation unit. - - NOTE(harlowja): during this pattern translation process any nested flows - will be converted into there equivalent subgraphs. This currently implies - that contained atoms in those nested flows, post-translation will no longer - be associated with there previously containing flow but instead will lose - this identity and what will remain is the logical constraints that there - contained flow mandated. In the future this may be changed so that this - association is not lost via the compilation process (since it can be - useful to retain this relationship). - """ - def compile(self, root): - graph = _Flattener(root).flatten() - if graph.number_of_nodes() == 0: - # Try to get a name attribute, otherwise just use the object - # string representation directly if that attribute does not exist. - name = getattr(root, 'name', root) - raise exc.Empty("Root container '%s' (%s) is empty." - % (name, type(root))) - return Compilation(graph) + @property + def hierarchy(self): + return self._hierarchy _RETRY_EDGE_DATA = { @@ -69,14 +50,15 @@ _RETRY_EDGE_DATA = { } -class _Flattener(object): - """Flattens a root item (task/flow) into a execution graph.""" +class PatternCompiler(object): + """Compiles a pattern (or task) into a compilation unit.""" def __init__(self, root, freeze=True): self._root = root - self._graph = None self._history = set() - self._freeze = bool(freeze) + self._freeze = freeze + self._lock = threading.Lock() + self._compilation = None def _add_new_edges(self, graph, nodes_from, nodes_to, edge_attrs): """Adds new edges from nodes to other nodes in the specified graph. @@ -93,72 +75,74 @@ class _Flattener(object): # if it's later modified that the same copy isn't modified. graph.add_edge(u, v, attr_dict=edge_attrs.copy()) - def _flatten(self, item): - functor = self._find_flattener(item) - if not functor: - raise TypeError("Unknown type requested to flatten: %s (%s)" - % (item, type(item))) + def _flatten(self, item, parent): + functor = self._find_flattener(item, parent) self._pre_item_flatten(item) - graph = functor(item) - self._post_item_flatten(item, graph) - return graph + graph, node = functor(item, parent) + self._post_item_flatten(item, graph, node) + return graph, node - def _find_flattener(self, item): + def _find_flattener(self, item, parent): """Locates the flattening function to use to flatten the given item.""" if isinstance(item, flow.Flow): return self._flatten_flow elif isinstance(item, task.BaseTask): return self._flatten_task elif isinstance(item, retry.Retry): - if len(self._history) == 1: - raise TypeError("Retry controller: %s (%s) must only be used" + if parent is None: + raise TypeError("Retry controller '%s' (%s) must only be used" " as a flow constructor parameter and not as a" " root component" % (item, type(item))) else: - # TODO(harlowja): we should raise this type error earlier - # instead of later since we should do this same check on add() - # calls, this makes the error more visible (instead of waiting - # until compile time). - raise TypeError("Retry controller: %s (%s) must only be used" + raise TypeError("Retry controller '%s' (%s) must only be used" " as a flow constructor parameter and not as a" " flow added component" % (item, type(item))) else: - return None + raise TypeError("Unknown item '%s' (%s) requested to flatten" + % (item, type(item))) def _connect_retry(self, retry, graph): graph.add_node(retry) - # All graph nodes that have no predecessors should depend on its retry - nodes_to = [n for n in graph.no_predecessors_iter() if n != retry] + # All nodes that have no predecessors should depend on this retry. + nodes_to = [n for n in graph.no_predecessors_iter() if n is not retry] self._add_new_edges(graph, [retry], nodes_to, _RETRY_EDGE_DATA) - # Add link to retry for each node of subgraph that hasn't - # a parent retry + # Add association for each node of graph that has no existing retry. for n in graph.nodes_iter(): - if n != retry and 'retry' not in graph.node[n]: + if n is not retry and 'retry' not in graph.node[n]: graph.node[n]['retry'] = retry - def _flatten_task(self, task): + def _flatten_task(self, task, parent): """Flattens a individual task.""" graph = gr.DiGraph(name=task.name) graph.add_node(task) - return graph + node = tr.Node(task) + if parent is not None: + parent.add(node) + return graph, node - def _flatten_flow(self, flow): - """Flattens a graph flow.""" + def _flatten_flow(self, flow, parent): + """Flattens a flow.""" graph = gr.DiGraph(name=flow.name) + node = tr.Node(flow) + if parent is not None: + parent.add(node) + if flow.retry is not None: + node.add(tr.Node(flow.retry)) - # Flatten all nodes into a single subgraph per node. - subgraph_map = {} + # Flatten all nodes into a single subgraph per item (and track origin + # item to its newly expanded graph). + subgraphs = {} for item in flow: - subgraph = self._flatten(item) - subgraph_map[item] = subgraph + subgraph = self._flatten(item, node)[0] + subgraphs[item] = subgraph graph = gr.merge_graphs([graph, subgraph]) - # Reconnect all node edges to their corresponding subgraphs. + # Reconnect all items edges to their corresponding subgraphs. for (u, v, attrs) in flow.iter_links(): - u_g = subgraph_map[u] - v_g = subgraph_map[v] + u_g = subgraphs[u] + v_g = subgraphs[v] if any(attrs.get(k) for k in ('invariant', 'manual', 'retry')): # Connect nodes with no predecessors in v to nodes with # no successors in u (thus maintaining the edge dependency). @@ -177,48 +161,57 @@ class _Flattener(object): if flow.retry is not None: self._connect_retry(flow.retry, graph) - return graph + return graph, node def _pre_item_flatten(self, item): """Called before a item is flattened; any pre-flattening actions.""" - if id(item) in self._history: - raise ValueError("Already flattened item: %s (%s), recursive" - " flattening not supported" % (item, id(item))) - self._history.add(id(item)) + if item in self._history: + raise ValueError("Already flattened item '%s' (%s), recursive" + " flattening is not supported" % (item, + type(item))) + self._history.add(item) - def _post_item_flatten(self, item, graph): - """Called before a item is flattened; any post-flattening actions.""" + def _post_item_flatten(self, item, graph, node): + """Called after a item is flattened; doing post-flattening actions.""" def _pre_flatten(self): - """Called before the flattening of the item starts.""" + """Called before the flattening of the root starts.""" self._history.clear() - def _post_flatten(self, graph): - """Called after the flattening of the item finishes successfully.""" + def _post_flatten(self, graph, node): + """Called after the flattening of the root finishes successfully.""" dup_names = misc.get_duplicate_keys(graph.nodes_iter(), key=lambda node: node.name) if dup_names: - dup_names = ', '.join(sorted(dup_names)) - raise exc.Duplicate("Atoms with duplicate names " - "found: %s" % (dup_names)) + raise exc.Duplicate( + "Atoms with duplicate names found: %s" % (sorted(dup_names))) + if graph.number_of_nodes() == 0: + raise exc.Empty("Root container '%s' (%s) is empty" + % (self._root, type(self._root))) self._history.clear() # NOTE(harlowja): this one can be expensive to calculate (especially # the cycle detection), so only do it if we know debugging is enabled # and not under all cases. if LOG.isEnabledFor(logging.DEBUG): - LOG.debug("Translated '%s' into a graph:", self._root) + LOG.debug("Translated '%s'", self._root) + LOG.debug("Graph:") for line in graph.pformat().splitlines(): # Indent it so that it's slightly offset from the above line. - LOG.debug(" %s", line) + LOG.debug(" %s", line) + LOG.debug("Hierarchy:") + for line in node.pformat().splitlines(): + # Indent it so that it's slightly offset from the above line. + LOG.debug(" %s", line) - def flatten(self): - """Flattens a item (a task or flow) into a single execution graph.""" - if self._graph is not None: - return self._graph - self._pre_flatten() - graph = self._flatten(self._root) - self._post_flatten(graph) - self._graph = graph - if self._freeze: - self._graph.freeze() - return self._graph + @lock_utils.locked + def compile(self): + """Compiles the contained item into a compiled equivalent.""" + if self._compilation is None: + self._pre_flatten() + graph, node = self._flatten(self._root, None) + self._post_flatten(graph, node) + if self._freeze: + graph.freeze() + node.freeze() + self._compilation = Compilation(graph, node) + return self._compilation diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index a5f587fd..cec73071 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -210,13 +210,13 @@ class ActionEngine(base.EngineBase): @misc.cachedproperty def _compiler(self): - return self._compiler_factory() + return self._compiler_factory(self._flow) @lock_utils.locked def compile(self): if self._compiled: return - self._compilation = self._compiler.compile(self._flow) + self._compilation = self._compiler.compile() self._runtime = runtime.Runtime(self._compilation, self.storage, self.task_notifier, diff --git a/taskflow/engines/action_engine/retry_action.py b/taskflow/engines/action_engine/retry_action.py index afdfb456..e4df5afa 100644 --- a/taskflow/engines/action_engine/retry_action.py +++ b/taskflow/engines/action_engine/retry_action.py @@ -27,13 +27,16 @@ SAVE_RESULT_STATES = (states.SUCCESS, states.FAILURE) class RetryAction(object): - def __init__(self, storage, notifier): + def __init__(self, storage, notifier, walker_factory): self._storage = storage self._notifier = notifier + self._walker_factory = walker_factory def _get_retry_args(self, retry): + scope_walker = self._walker_factory(retry) kwargs = self._storage.fetch_mapped_args(retry.rebind, - atom_name=retry.name) + atom_name=retry.name, + scope_walker=scope_walker) kwargs['history'] = self._storage.get_retry_history(retry.name) return kwargs diff --git a/taskflow/engines/action_engine/runtime.py b/taskflow/engines/action_engine/runtime.py index 90913b99..c0c58367 100644 --- a/taskflow/engines/action_engine/runtime.py +++ b/taskflow/engines/action_engine/runtime.py @@ -18,6 +18,7 @@ from taskflow.engines.action_engine import analyzer as ca from taskflow.engines.action_engine import executor as ex from taskflow.engines.action_engine import retry_action as ra from taskflow.engines.action_engine import runner as ru +from taskflow.engines.action_engine import scopes as sc from taskflow.engines.action_engine import task_action as ta from taskflow import exceptions as excp from taskflow import retry as retry_atom @@ -66,12 +67,18 @@ class Runtime(object): @misc.cachedproperty def retry_action(self): - return ra.RetryAction(self.storage, self._task_notifier) + return ra.RetryAction(self._storage, self._task_notifier, + lambda atom: sc.ScopeWalker(self.compilation, + atom, + names_only=True)) @misc.cachedproperty def task_action(self): - return ta.TaskAction(self.storage, self._task_executor, - self._task_notifier) + return ta.TaskAction(self._storage, self._task_executor, + self._task_notifier, + lambda atom: sc.ScopeWalker(self.compilation, + atom, + names_only=True)) def reset_nodes(self, nodes, state=st.PENDING, intention=st.EXECUTE): for node in nodes: @@ -81,7 +88,7 @@ class Runtime(object): elif isinstance(node, retry_atom.Retry): self.retry_action.change_state(node, state) else: - raise TypeError("Unknown how to reset node %s, %s" + raise TypeError("Unknown how to reset atom '%s' (%s)" % (node, type(node))) if intention: self.storage.set_atom_intention(node.name, intention) @@ -209,7 +216,7 @@ class Scheduler(object): elif isinstance(node, retry_atom.Retry): return self._schedule_retry(node) else: - raise TypeError("Unknown how to schedule node %s, %s" + raise TypeError("Unknown how to schedule atom '%s' (%s)" % (node, type(node))) def _schedule_retry(self, retry): diff --git a/taskflow/engines/action_engine/scopes.py b/taskflow/engines/action_engine/scopes.py new file mode 100644 index 00000000..f1dd49d1 --- /dev/null +++ b/taskflow/engines/action_engine/scopes.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + +from taskflow import atom as atom_type +from taskflow import flow as flow_type + +LOG = logging.getLogger(__name__) + + +def _extract_atoms(node, idx=-1): + # Always go left to right, since right to left is the pattern order + # and we want to go backwards and not forwards through that ordering... + if idx == -1: + children_iter = node.reverse_iter() + else: + children_iter = reversed(node[0:idx]) + atoms = [] + for child in children_iter: + if isinstance(child.item, flow_type.Flow): + atoms.extend(_extract_atoms(child)) + elif isinstance(child.item, atom_type.Atom): + atoms.append(child.item) + else: + raise TypeError( + "Unknown extraction item '%s' (%s)" % (child.item, + type(child.item))) + return atoms + + +class ScopeWalker(object): + """Walks through the scopes of a atom using a engines compilation. + + This will walk the visible scopes that are accessible for the given + atom, which can be used by some external entity in some meaningful way, + for example to find dependent values... + """ + + def __init__(self, compilation, atom, names_only=False): + self._node = compilation.hierarchy.find(atom) + if self._node is None: + raise ValueError("Unable to find atom '%s' in compilation" + " hierarchy" % atom) + self._atom = atom + self._graph = compilation.execution_graph + self._names_only = names_only + + def __iter__(self): + """Iterates over the visible scopes. + + How this works is the following: + + We find all the possible predecessors of the given atom, this is useful + since we know they occurred before this atom but it doesn't tell us + the corresponding scope *level* that each predecessor was created in, + so we need to find this information. + + For that information we consult the location of the atom ``Y`` in the + node hierarchy. We lookup in a reverse order the parent ``X`` of ``Y`` + and traverse backwards from the index in the parent where ``Y`` + occurred, all children in ``X`` that we encounter in this backwards + search (if a child is a flow itself, its atom contents will be + expanded) will be assumed to be at the same scope. This is then a + *potential* single scope, to make an *actual* scope we remove the items + from the *potential* scope that are not predecessors of ``Y`` to form + the *actual* scope. + + Then for additional scopes we continue up the tree, by finding the + parent of ``X`` (lets call it ``Z``) and perform the same operation, + going through the children in a reverse manner from the index in + parent ``Z`` where ``X`` was located. This forms another *potential* + scope which we provide back as an *actual* scope after reducing the + potential set by the predecessors of ``Y``. We then repeat this process + until we no longer have any parent nodes (aka have reached the top of + the tree) or we run out of predecessors. + """ + predecessors = set(self._graph.bfs_predecessors_iter(self._atom)) + last = self._node + for parent in self._node.path_iter(include_self=False): + if not predecessors: + break + last_idx = parent.index(last.item) + visible = [] + for a in _extract_atoms(parent, idx=last_idx): + if a in predecessors: + predecessors.remove(a) + if not self._names_only: + visible.append(a) + else: + visible.append(a.name) + if LOG.isEnabledFor(logging.DEBUG): + if not self._names_only: + visible_names = [a.name for a in visible] + else: + visible_names = visible + # TODO(harlowja): we should likely use a created TRACE level + # for this kind of *very* verbose information; otherwise the + # cinder and other folks are going to complain that there + # debug logs are full of not so useful information (it is + # useful to taskflow debugging...). + LOG.debug("Scope visible to '%s' (limited by parent '%s' index" + " < %s) is: %s", self._atom, parent.item.name, + last_idx, visible_names) + yield visible + last = parent diff --git a/taskflow/engines/action_engine/task_action.py b/taskflow/engines/action_engine/task_action.py index a07ded79..3503df7c 100644 --- a/taskflow/engines/action_engine/task_action.py +++ b/taskflow/engines/action_engine/task_action.py @@ -26,10 +26,11 @@ SAVE_RESULT_STATES = (states.SUCCESS, states.FAILURE) class TaskAction(object): - def __init__(self, storage, task_executor, notifier): + def __init__(self, storage, task_executor, notifier, walker_factory): self._storage = storage self._task_executor = task_executor self._notifier = notifier + self._walker_factory = walker_factory def _is_identity_transition(self, state, task, progress): if state in SAVE_RESULT_STATES: @@ -81,8 +82,10 @@ class TaskAction(object): def schedule_execution(self, task): self.change_state(task, states.RUNNING, progress=0.0) + scope_walker = self._walker_factory(task) kwargs = self._storage.fetch_mapped_args(task.rebind, - atom_name=task.name) + atom_name=task.name, + scope_walker=scope_walker) task_uuid = self._storage.get_atom_uuid(task.name) return self._task_executor.execute_task(task, task_uuid, kwargs, self._on_update_progress) @@ -96,8 +99,10 @@ class TaskAction(object): def schedule_reversion(self, task): self.change_state(task, states.REVERTING, progress=0.0) + scope_walker = self._walker_factory(task) kwargs = self._storage.fetch_mapped_args(task.rebind, - atom_name=task.name) + atom_name=task.name, + scope_walker=scope_walker) task_uuid = self._storage.get_atom_uuid(task.name) task_result = self._storage.get(task.name) failures = self._storage.get_failures() diff --git a/taskflow/patterns/graph_flow.py b/taskflow/patterns/graph_flow.py index 9717b211..658e051c 100644 --- a/taskflow/patterns/graph_flow.py +++ b/taskflow/patterns/graph_flow.py @@ -161,7 +161,7 @@ class Flow(flow.Flow): return self._get_subgraph().number_of_nodes() def __iter__(self): - for n in self._get_subgraph().nodes_iter(): + for n in self._get_subgraph().topological_sort(): yield n def iter_links(self): diff --git a/taskflow/storage.py b/taskflow/storage.py index 31a8868f..bcc2b157 100644 --- a/taskflow/storage.py +++ b/taskflow/storage.py @@ -31,6 +31,78 @@ from taskflow.utils import reflection LOG = logging.getLogger(__name__) STATES_WITH_RESULTS = (states.SUCCESS, states.REVERTING, states.FAILURE) +# TODO(harlowja): do this better (via a singleton or something else...) +_TRANSIENT_PROVIDER = object() + +# NOTE(harlowja): Perhaps the container is a dictionary-like object and that +# key does not exist (key error), or the container is a tuple/list and a +# non-numeric key is being requested (index error), or there was no container +# and an attempt to index into none/other unsubscriptable type is being +# requested (type error). +# +# Overall this (along with the item_from* functions) try to handle the vast +# majority of wrong indexing operations on the wrong/invalid types so that we +# can fail extraction during lookup or emit warning on result reception... +_EXTRACTION_EXCEPTIONS = (IndexError, KeyError, ValueError, TypeError) + + +class _Provider(object): + """A named symbol provider that produces a output at the given index.""" + + def __init__(self, name, index): + self.name = name + self.index = index + + def __repr__(self): + # TODO(harlowja): clean this up... + if self.name is _TRANSIENT_PROVIDER: + base = " index. If index is None, the whole result will have this name; else, only part of it, result[index]. """ - if not mapping: - return - self._result_mappings[atom_name] = mapping - for name, index in six.iteritems(mapping): - entries = self._reverse_mapping.setdefault(name, []) + provider_mapping = self._result_mappings.setdefault(provider_name, {}) + if mapping: + provider_mapping.update(mapping) + # Ensure the reverse mapping/index is updated (for faster lookups). + for name, index in six.iteritems(provider_mapping): + entries = self._reverse_mapping.setdefault(name, []) + provider = _Provider(provider_name, index) + if provider not in entries: + entries.append(provider) - # NOTE(imelnikov): We support setting same result mapping for - # the same atom twice (e.g when we are injecting 'a' and then - # injecting 'a' again), so we should not log warning below in - # that case and we should have only one item for each pair - # (atom_name, index) in entries. It should be put to the end of - # entries list because order matters on fetching. - try: - entries.remove((atom_name, index)) - except ValueError: - pass - - entries.append((atom_name, index)) - if len(entries) > 1: - LOG.warning("Multiple provider mappings being created for %r", - name) - - def fetch(self, name): - """Fetch a named atoms result.""" + def fetch(self, name, many_handler=None): + """Fetch a named result.""" + # By default we just return the first of many (unless provided + # a different callback that can translate many results into something + # more meaningful). + if many_handler is None: + many_handler = lambda values: values[0] with self._lock.read_lock(): try: - indexes = self._reverse_mapping[name] + providers = self._reverse_mapping[name] except KeyError: - raise exceptions.NotFound("Name %r is not mapped" % name) - # Return the first one that is found. - for (atom_name, index) in reversed(indexes): - if not atom_name: - results = self._transients + raise exceptions.NotFound("Name %r is not mapped as a" + " produced output by any" + " providers" % name) + values = [] + for provider in providers: + if provider.name is _TRANSIENT_PROVIDER: + values.append(_item_from_single(provider, + self._transients, name)) else: - results = self._get(atom_name, only_last=True) - try: - return misc.item_from(results, index, name) - except exceptions.NotFound: - pass - raise exceptions.NotFound("Unable to find result %r" % name) + try: + container = self._get(provider.name, only_last=True) + except exceptions.NotFound: + pass + else: + values.append(_item_from_single(provider, + container, name)) + if not values: + raise exceptions.NotFound("Unable to find result %r," + " searched %s" % (name, providers)) + else: + return many_handler(values) def fetch_all(self): - """Fetch all named atom results known so far. + """Fetch all named results known so far. - Should be used for debugging and testing purposes mostly. + NOTE(harlowja): should be used for debugging and testing purposes. """ + def many_handler(values): + if len(values) > 1: + return values + return values[0] with self._lock.read_lock(): results = {} - for name in self._reverse_mapping: + for name in six.iterkeys(self._reverse_mapping): try: - results[name] = self.fetch(name) + results[name] = self.fetch(name, many_handler=many_handler) except exceptions.NotFound: pass return results - def fetch_mapped_args(self, args_mapping, atom_name=None): - """Fetch arguments for an atom using an atoms arguments mapping.""" + def fetch_mapped_args(self, args_mapping, + atom_name=None, scope_walker=None): + """Fetch arguments for an atom using an atoms argument mapping.""" + + def _get_results(looking_for, provider): + """Gets the results saved for a given provider.""" + try: + return self._get(provider.name, only_last=True) + except exceptions.NotFound as e: + raise exceptions.NotFound( + "Expected to be able to find output %r produced" + " by %s but was unable to get at that providers" + " results" % (looking_for, provider), e) + + def _locate_providers(looking_for, possible_providers): + """Finds the accessible providers.""" + default_providers = [] + for p in possible_providers: + if p.name is _TRANSIENT_PROVIDER: + default_providers.append((p, self._transients)) + if p.name == self.injector_name: + default_providers.append((p, _get_results(looking_for, p))) + if default_providers: + return default_providers + if scope_walker is not None: + scope_iter = iter(scope_walker) + else: + scope_iter = iter([]) + for atom_names in scope_iter: + if not atom_names: + continue + providers = [] + for p in possible_providers: + if p.name in atom_names: + providers.append((p, _get_results(looking_for, p))) + if providers: + return providers + return [] + with self._lock.read_lock(): - injected_args = {} + if atom_name and atom_name not in self._atom_name_to_uuid: + raise exceptions.NotFound("Unknown atom name: %s" % atom_name) + if not args_mapping: + return {} + # The order of lookup is the following: + # + # 1. Injected atom specific arguments. + # 2. Transient injected arguments. + # 3. Non-transient injected arguments. + # 4. First scope visited group that produces the named result. + # a). The first of that group that actually provided the name + # result is selected (if group size is greater than one). + # + # Otherwise: blowup! (this will also happen if reading or + # extracting an expected result fails, since it is better to fail + # on lookup then provide invalid data from the wrong provider) if atom_name: injected_args = self._injected_args.get(atom_name, {}) + else: + injected_args = {} mapped_args = {} - for key, name in six.iteritems(args_mapping): + for (bound_name, name) in six.iteritems(args_mapping): + # TODO(harlowja): This logging information may be to verbose + # even for DEBUG mode, let's see if we can maybe in the future + # add a TRACE mode or something else if people complain... + if LOG.isEnabledFor(logging.DEBUG): + if atom_name: + LOG.debug("Looking for %r <= %r for atom named: %s", + bound_name, name, atom_name) + else: + LOG.debug("Looking for %r <= %r", bound_name, name) if name in injected_args: - mapped_args[key] = injected_args[name] + value = injected_args[name] + mapped_args[bound_name] = value + LOG.debug("Matched %r <= %r to %r (from injected values)", + bound_name, name, value) else: - mapped_args[key] = self.fetch(name) + try: + possible_providers = self._reverse_mapping[name] + except KeyError: + raise exceptions.NotFound("Name %r is not mapped as a" + " produced output by any" + " providers" % name) + # Reduce the possible providers to one that are allowed. + providers = _locate_providers(name, possible_providers) + if not providers: + raise exceptions.NotFound( + "Mapped argument %r <= %r was not produced" + " by any accessible provider (%s possible" + " providers were scanned)" + % (bound_name, name, len(possible_providers))) + provider, value = _item_from_first_of(providers, name) + mapped_args[bound_name] = value + LOG.debug("Matched %r <= %r to %r (from %s)", + bound_name, name, value, provider) return mapped_args def set_flow_state(self, state): diff --git a/taskflow/tests/unit/action_engine/test_compile.py b/taskflow/tests/unit/action_engine/test_compile.py index 7207468e..63b3c0b0 100644 --- a/taskflow/tests/unit/action_engine/test_compile.py +++ b/taskflow/tests/unit/action_engine/test_compile.py @@ -27,21 +27,25 @@ from taskflow.tests import utils as test_utils class PatternCompileTest(test.TestCase): def test_task(self): task = test_utils.DummyTask(name='a') - compilation = compiler.PatternCompiler().compile(task) + compilation = compiler.PatternCompiler(task).compile() g = compilation.execution_graph self.assertEqual(list(g.nodes()), [task]) self.assertEqual(list(g.edges()), []) def test_retry(self): r = retry.AlwaysRevert('r1') - msg_regex = "^Retry controller: .* must only be used .*" + msg_regex = "^Retry controller .* must only be used .*" self.assertRaisesRegexp(TypeError, msg_regex, - compiler.PatternCompiler().compile, r) + compiler.PatternCompiler(r).compile) def test_wrong_object(self): - msg_regex = '^Unknown type requested to flatten' + msg_regex = '^Unknown item .* requested to flatten' self.assertRaisesRegexp(TypeError, msg_regex, - compiler.PatternCompiler().compile, 42) + compiler.PatternCompiler(42).compile) + + def test_empty(self): + flo = lf.Flow("test") + self.assertRaises(exc.Empty, compiler.PatternCompiler(flo).compile) def test_linear(self): a, b, c, d = test_utils.make_many(4) @@ -51,7 +55,7 @@ class PatternCompileTest(test.TestCase): sflo.add(d) flo.add(sflo) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(4, len(g)) @@ -69,13 +73,13 @@ class PatternCompileTest(test.TestCase): flo.add(a, b, c) flo.add(flo) self.assertRaises(ValueError, - compiler.PatternCompiler().compile, flo) + compiler.PatternCompiler(flo).compile) def test_unordered(self): a, b, c, d = test_utils.make_many(4) flo = uf.Flow("test") flo.add(a, b, c, d) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(4, len(g)) self.assertEqual(0, g.number_of_edges()) @@ -92,7 +96,7 @@ class PatternCompileTest(test.TestCase): flo2.add(c, d) flo.add(flo2) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(4, len(g)) @@ -116,7 +120,7 @@ class PatternCompileTest(test.TestCase): flo2.add(c, d) flo.add(flo2) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(4, len(g)) for n in [a, b]: @@ -138,7 +142,7 @@ class PatternCompileTest(test.TestCase): uf.Flow('ut').add(b, c), d) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(4, len(g)) self.assertItemsEqual(g.edges(), [ @@ -153,7 +157,7 @@ class PatternCompileTest(test.TestCase): flo = gf.Flow("test") flo.add(a, b, c, d) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(4, len(g)) self.assertEqual(0, g.number_of_edges()) @@ -167,7 +171,7 @@ class PatternCompileTest(test.TestCase): flo2.add(e, f, g) flo.add(flo2) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() graph = compilation.execution_graph self.assertEqual(7, len(graph)) self.assertItemsEqual(graph.edges(data=True), [ @@ -184,7 +188,7 @@ class PatternCompileTest(test.TestCase): flo2.add(e, f, g) flo.add(flo2) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(7, len(g)) self.assertEqual(0, g.number_of_edges()) @@ -197,7 +201,7 @@ class PatternCompileTest(test.TestCase): flo.link(b, c) flo.link(c, d) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(4, len(g)) self.assertItemsEqual(g.edges(data=True), [ @@ -213,7 +217,7 @@ class PatternCompileTest(test.TestCase): b = test_utils.ProvidesRequiresTask('b', provides=[], requires=['x']) flo = gf.Flow("test").add(a, b) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(2, len(g)) self.assertItemsEqual(g.edges(data=True), [ @@ -231,7 +235,7 @@ class PatternCompileTest(test.TestCase): lf.Flow("test2").add(b, c) ) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(3, len(g)) self.assertItemsEqual(g.edges(data=True), [ @@ -250,7 +254,7 @@ class PatternCompileTest(test.TestCase): lf.Flow("test2").add(b, c) ) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(3, len(g)) self.assertItemsEqual(g.edges(data=True), [ @@ -267,7 +271,7 @@ class PatternCompileTest(test.TestCase): ) self.assertRaisesRegexp(exc.Duplicate, '^Atoms with duplicate names', - compiler.PatternCompiler().compile, flo) + compiler.PatternCompiler(flo).compile) def test_checks_for_dups_globally(self): flo = gf.Flow("test").add( @@ -275,25 +279,25 @@ class PatternCompileTest(test.TestCase): gf.Flow("int2").add(test_utils.DummyTask(name="a"))) self.assertRaisesRegexp(exc.Duplicate, '^Atoms with duplicate names', - compiler.PatternCompiler().compile, flo) + compiler.PatternCompiler(flo).compile) def test_retry_in_linear_flow(self): flo = lf.Flow("test", retry.AlwaysRevert("c")) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(1, len(g)) self.assertEqual(0, g.number_of_edges()) def test_retry_in_unordered_flow(self): flo = uf.Flow("test", retry.AlwaysRevert("c")) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(1, len(g)) self.assertEqual(0, g.number_of_edges()) def test_retry_in_graph_flow(self): flo = gf.Flow("test", retry.AlwaysRevert("c")) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(1, len(g)) self.assertEqual(0, g.number_of_edges()) @@ -302,7 +306,7 @@ class PatternCompileTest(test.TestCase): c1 = retry.AlwaysRevert("c1") c2 = retry.AlwaysRevert("c2") flo = lf.Flow("test", c1).add(lf.Flow("test2", c2)) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(2, len(g)) @@ -317,7 +321,7 @@ class PatternCompileTest(test.TestCase): c = retry.AlwaysRevert("c") a, b = test_utils.make_many(2) flo = lf.Flow("test", c).add(a, b) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(3, len(g)) @@ -335,7 +339,7 @@ class PatternCompileTest(test.TestCase): c = retry.AlwaysRevert("c") a, b = test_utils.make_many(2) flo = uf.Flow("test", c).add(a, b) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(3, len(g)) @@ -353,7 +357,7 @@ class PatternCompileTest(test.TestCase): r = retry.AlwaysRevert("cp") a, b, c = test_utils.make_many(3) flo = gf.Flow("test", r).add(a, b, c).link(b, c) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(4, len(g)) @@ -377,7 +381,7 @@ class PatternCompileTest(test.TestCase): a, lf.Flow("test", c2).add(b, c), d) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(6, len(g)) @@ -402,7 +406,7 @@ class PatternCompileTest(test.TestCase): a, lf.Flow("test").add(b, c), d) - compilation = compiler.PatternCompiler().compile(flo) + compilation = compiler.PatternCompiler(flo).compile() g = compilation.execution_graph self.assertEqual(5, len(g)) diff --git a/taskflow/tests/unit/action_engine/test_runner.py b/taskflow/tests/unit/action_engine/test_runner.py index 2e18f6b6..82440fc5 100644 --- a/taskflow/tests/unit/action_engine/test_runner.py +++ b/taskflow/tests/unit/action_engine/test_runner.py @@ -33,7 +33,7 @@ from taskflow.utils import persistence_utils as pu class _RunnerTestMixin(object): def _make_runtime(self, flow, initial_state=None): - compilation = compiler.PatternCompiler().compile(flow) + compilation = compiler.PatternCompiler(flow).compile() flow_detail = pu.create_flow_detail(flow) store = storage.SingleThreadedStorage(flow_detail) # This ensures the tasks exist in storage... diff --git a/taskflow/tests/unit/test_action_engine_scoping.py b/taskflow/tests/unit/test_action_engine_scoping.py new file mode 100644 index 00000000..e2de763f --- /dev/null +++ b/taskflow/tests/unit/test_action_engine_scoping.py @@ -0,0 +1,248 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from taskflow.engines.action_engine import compiler +from taskflow.engines.action_engine import scopes as sc +from taskflow.patterns import graph_flow as gf +from taskflow.patterns import linear_flow as lf +from taskflow.patterns import unordered_flow as uf +from taskflow import test +from taskflow.tests import utils as test_utils + + +def _get_scopes(compilation, atom, names_only=True): + walker = sc.ScopeWalker(compilation, atom, names_only=names_only) + return list(iter(walker)) + + +class LinearScopingTest(test.TestCase): + def test_unknown(self): + r = lf.Flow("root") + r_1 = test_utils.TaskOneReturn("root.1") + r.add(r_1) + + r_2 = test_utils.TaskOneReturn("root.2") + c = compiler.PatternCompiler(r).compile() + self.assertRaises(ValueError, _get_scopes, c, r_2) + + def test_empty(self): + r = lf.Flow("root") + r_1 = test_utils.TaskOneReturn("root.1") + r.add(r_1) + + c = compiler.PatternCompiler(r).compile() + self.assertIn(r_1, c.execution_graph) + self.assertIsNotNone(c.hierarchy.find(r_1)) + + walker = sc.ScopeWalker(c, r_1) + scopes = list(walker) + self.assertEqual([], scopes) + + def test_single_prior_linear(self): + r = lf.Flow("root") + r_1 = test_utils.TaskOneReturn("root.1") + r_2 = test_utils.TaskOneReturn("root.2") + r.add(r_1, r_2) + + c = compiler.PatternCompiler(r).compile() + for a in r: + self.assertIn(a, c.execution_graph) + self.assertIsNotNone(c.hierarchy.find(a)) + + self.assertEqual([], _get_scopes(c, r_1)) + self.assertEqual([['root.1']], _get_scopes(c, r_2)) + + def test_nested_prior_linear(self): + r = lf.Flow("root") + r.add(test_utils.TaskOneReturn("root.1"), + test_utils.TaskOneReturn("root.2")) + sub_r = lf.Flow("subroot") + sub_r_1 = test_utils.TaskOneReturn("subroot.1") + sub_r.add(sub_r_1) + r.add(sub_r) + + c = compiler.PatternCompiler(r).compile() + self.assertEqual([[], ['root.2', 'root.1']], _get_scopes(c, sub_r_1)) + + def test_nested_prior_linear_begin_middle_end(self): + r = lf.Flow("root") + begin_r = test_utils.TaskOneReturn("root.1") + r.add(begin_r, test_utils.TaskOneReturn("root.2")) + middle_r = test_utils.TaskOneReturn("root.3") + r.add(middle_r) + sub_r = lf.Flow("subroot") + sub_r.add(test_utils.TaskOneReturn("subroot.1"), + test_utils.TaskOneReturn("subroot.2")) + r.add(sub_r) + end_r = test_utils.TaskOneReturn("root.4") + r.add(end_r) + + c = compiler.PatternCompiler(r).compile() + + self.assertEqual([], _get_scopes(c, begin_r)) + self.assertEqual([['root.2', 'root.1']], _get_scopes(c, middle_r)) + self.assertEqual([['subroot.2', 'subroot.1', 'root.3', 'root.2', + 'root.1']], _get_scopes(c, end_r)) + + +class GraphScopingTest(test.TestCase): + def test_dependent(self): + r = gf.Flow("root") + + customer = test_utils.ProvidesRequiresTask("customer", + provides=['dog'], + requires=[]) + washer = test_utils.ProvidesRequiresTask("washer", + requires=['dog'], + provides=['wash']) + dryer = test_utils.ProvidesRequiresTask("dryer", + requires=['dog', 'wash'], + provides=['dry_dog']) + shaved = test_utils.ProvidesRequiresTask("shaver", + requires=['dry_dog'], + provides=['shaved_dog']) + happy_customer = test_utils.ProvidesRequiresTask( + "happy_customer", requires=['shaved_dog'], provides=['happiness']) + + r.add(customer, washer, dryer, shaved, happy_customer) + + c = compiler.PatternCompiler(r).compile() + + self.assertEqual([], _get_scopes(c, customer)) + self.assertEqual([['washer', 'customer']], _get_scopes(c, dryer)) + self.assertEqual([['shaver', 'dryer', 'washer', 'customer']], + _get_scopes(c, happy_customer)) + + def test_no_visible(self): + r = gf.Flow("root") + atoms = [] + for i in range(0, 10): + atoms.append(test_utils.TaskOneReturn("root.%s" % i)) + r.add(*atoms) + + c = compiler.PatternCompiler(r).compile() + for a in atoms: + self.assertEqual([], _get_scopes(c, a)) + + def test_nested(self): + r = gf.Flow("root") + + r_1 = test_utils.TaskOneReturn("root.1") + r_2 = test_utils.TaskOneReturn("root.2") + r.add(r_1, r_2) + r.link(r_1, r_2) + + subroot = gf.Flow("subroot") + subroot_r_1 = test_utils.TaskOneReturn("subroot.1") + subroot_r_2 = test_utils.TaskOneReturn("subroot.2") + subroot.add(subroot_r_1, subroot_r_2) + subroot.link(subroot_r_1, subroot_r_2) + + r.add(subroot) + r_3 = test_utils.TaskOneReturn("root.3") + r.add(r_3) + r.link(r_2, r_3) + + c = compiler.PatternCompiler(r).compile() + self.assertEqual([], _get_scopes(c, r_1)) + self.assertEqual([['root.1']], _get_scopes(c, r_2)) + self.assertEqual([['root.2', 'root.1']], _get_scopes(c, r_3)) + + self.assertEqual([], _get_scopes(c, subroot_r_1)) + self.assertEqual([['subroot.1']], _get_scopes(c, subroot_r_2)) + + +class UnorderedScopingTest(test.TestCase): + def test_no_visible(self): + r = uf.Flow("root") + atoms = [] + for i in range(0, 10): + atoms.append(test_utils.TaskOneReturn("root.%s" % i)) + r.add(*atoms) + c = compiler.PatternCompiler(r).compile() + for a in atoms: + self.assertEqual([], _get_scopes(c, a)) + + +class MixedPatternScopingTest(test.TestCase): + def test_graph_linear_scope(self): + r = gf.Flow("root") + r_1 = test_utils.TaskOneReturn("root.1") + r_2 = test_utils.TaskOneReturn("root.2") + r.add(r_1, r_2) + r.link(r_1, r_2) + + s = lf.Flow("subroot") + s_1 = test_utils.TaskOneReturn("subroot.1") + s_2 = test_utils.TaskOneReturn("subroot.2") + s.add(s_1, s_2) + r.add(s) + + t = gf.Flow("subroot2") + t_1 = test_utils.TaskOneReturn("subroot2.1") + t_2 = test_utils.TaskOneReturn("subroot2.2") + t.add(t_1, t_2) + t.link(t_1, t_2) + r.add(t) + r.link(s, t) + + c = compiler.PatternCompiler(r).compile() + self.assertEqual([], _get_scopes(c, r_1)) + self.assertEqual([['root.1']], _get_scopes(c, r_2)) + self.assertEqual([], _get_scopes(c, s_1)) + self.assertEqual([['subroot.1']], _get_scopes(c, s_2)) + self.assertEqual([[], ['subroot.2', 'subroot.1']], + _get_scopes(c, t_1)) + self.assertEqual([["subroot2.1"], ['subroot.2', 'subroot.1']], + _get_scopes(c, t_2)) + + def test_linear_unordered_scope(self): + r = lf.Flow("root") + r_1 = test_utils.TaskOneReturn("root.1") + r_2 = test_utils.TaskOneReturn("root.2") + r.add(r_1, r_2) + + u = uf.Flow("subroot") + atoms = [] + for i in range(0, 5): + atoms.append(test_utils.TaskOneReturn("subroot.%s" % i)) + u.add(*atoms) + r.add(u) + + r_3 = test_utils.TaskOneReturn("root.3") + r.add(r_3) + + c = compiler.PatternCompiler(r).compile() + + self.assertEqual([], _get_scopes(c, r_1)) + self.assertEqual([['root.1']], _get_scopes(c, r_2)) + for a in atoms: + self.assertEqual([[], ['root.2', 'root.1']], _get_scopes(c, a)) + + scope = _get_scopes(c, r_3) + self.assertEqual(1, len(scope)) + first_root = 0 + for i, n in enumerate(scope[0]): + if n.startswith('root.'): + first_root = i + break + first_subroot = 0 + for i, n in enumerate(scope[0]): + if n.startswith('subroot.'): + first_subroot = i + break + self.assertGreater(first_subroot, first_root) + self.assertEqual(scope[0][-2:], ['root.2', 'root.1']) diff --git a/taskflow/tests/unit/test_storage.py b/taskflow/tests/unit/test_storage.py index 001cba97..8060f28e 100644 --- a/taskflow/tests/unit/test_storage.py +++ b/taskflow/tests/unit/test_storage.py @@ -454,23 +454,6 @@ class StorageTestMixin(object): self.assertRaisesRegexp(exceptions.NotFound, '^Unable to find result', s.fetch, 'b') - @mock.patch.object(storage.LOG, 'warning') - def test_multiple_providers_are_checked(self, mocked_warning): - s = self._get_storage() - s.ensure_task('my task', result_mapping={'result': 'key'}) - self.assertEqual(mocked_warning.mock_calls, []) - s.ensure_task('my other task', result_mapping={'result': 'key'}) - mocked_warning.assert_called_once_with( - mock.ANY, 'result') - - @mock.patch.object(storage.LOG, 'warning') - def test_multiple_providers_with_inject_are_checked(self, mocked_warning): - s = self._get_storage() - s.inject({'result': 'DONE'}) - self.assertEqual(mocked_warning.mock_calls, []) - s.ensure_task('my other task', result_mapping={'result': 'key'}) - mocked_warning.assert_called_once_with(mock.ANY, 'result') - def test_ensure_retry(self): s = self._get_storage() s.ensure_retry('my retry') diff --git a/taskflow/types/tree.py b/taskflow/types/tree.py index 41369b04..e4fde5de 100644 --- a/taskflow/types/tree.py +++ b/taskflow/types/tree.py @@ -166,6 +166,11 @@ class Node(object): for c in self._children: yield c + def reverse_iter(self): + """Iterates over the direct children of this node (left->right).""" + for c in reversed(self._children): + yield c + def index(self, item): """Finds the child index of a given item, searchs in added order.""" index_at = None diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 8e5e1921..fbdc1530 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -257,24 +257,6 @@ def sequence_minus(seq1, seq2): return result -def item_from(container, index, name=None): - """Attempts to fetch a index/key from a given container.""" - if index is None: - return container - try: - return container[index] - except (IndexError, KeyError, ValueError, TypeError): - # NOTE(harlowja): Perhaps the container is a dictionary-like object - # and that key does not exist (key error), or the container is a - # tuple/list and a non-numeric key is being requested (index error), - # or there was no container and an attempt to index into none/other - # unsubscriptable type is being requested (type error). - if name is None: - name = index - raise exc.NotFound("Unable to find %r in container %s" - % (name, container)) - - def get_duplicate_keys(iterable, key=None): if key is not None: iterable = six.moves.map(key, iterable) @@ -399,8 +381,8 @@ def ensure_tree(path): """ try: os.makedirs(path) - except OSError as exc: - if exc.errno == errno.EEXIST: + except OSError as e: + if e.errno == errno.EEXIST: if not os.path.isdir(path): raise else: From 2339bacaf7edd9a781267cf8ca38b8639f34137b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 18 Jul 2014 12:49:39 -0700 Subject: [PATCH 013/240] Relax the linear flow symbol constraints In order to make it possible to have a symbol tree we need to relax and remove the constraints that are being imposed by the linear constraints and later move those constraint checks and validations into the engines compilation stage. Part of blueprint taskflow-improved-scoping Change-Id: I6efdc821ff991e83572d89f56be5c678d007f9f8 --- taskflow/patterns/linear_flow.py | 33 ++---------------- .../tests/unit/patterns/test_linear_flow.py | 19 ----------- taskflow/tests/unit/test_flow_dependencies.py | 34 ------------------- 3 files changed, 2 insertions(+), 84 deletions(-) diff --git a/taskflow/patterns/linear_flow.py b/taskflow/patterns/linear_flow.py index 14799b82..77559231 100644 --- a/taskflow/patterns/linear_flow.py +++ b/taskflow/patterns/linear_flow.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -from taskflow import exceptions from taskflow import flow @@ -22,14 +21,11 @@ _LINK_METADATA = {'invariant': True} class Flow(flow.Flow): - """Linear Flow pattern. + """Linear flow pattern. A linear (potentially nested) flow of *tasks/flows* that can be applied in order as one unit and rolled back as one unit using the reverse order that the *tasks/flows* have been applied in. - - NOTE(imelnikov): Tasks/flows contained in this linear flow must not - depend on outputs (provided names/values) of tasks/flows that follow it. """ def __init__(self, name, retry=None): @@ -38,32 +34,7 @@ class Flow(flow.Flow): def add(self, *items): """Adds a given task/tasks/flow/flows to this flow.""" - if not items: - return self - - # NOTE(imelnikov): we add item to the end of flow, so it should - # not provide anything previous items of the flow require. - requires = self.requires - provides = self.provides - for item in items: - requires |= item.requires - out_of_order = requires & item.provides - if out_of_order: - raise exceptions.DependencyFailure( - "%(item)s provides %(oo)s that are required " - "by previous item(s) of linear flow %(flow)s" - % dict(item=item.name, flow=self.name, - oo=sorted(out_of_order))) - same_provides = provides & item.provides - if same_provides: - raise exceptions.DependencyFailure( - "%(item)s provides %(value)s but is already being" - " provided by %(flow)s and duplicate producers" - " are disallowed" - % dict(item=item.name, flow=self.name, - value=sorted(same_provides))) - provides |= item.provides - + items = [i for i in items if i not in self._children] self._children.extend(items) return self diff --git a/taskflow/tests/unit/patterns/test_linear_flow.py b/taskflow/tests/unit/patterns/test_linear_flow.py index a0dbd0d7..23f891a8 100644 --- a/taskflow/tests/unit/patterns/test_linear_flow.py +++ b/taskflow/tests/unit/patterns/test_linear_flow.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -from taskflow import exceptions as exc from taskflow.patterns import linear_flow as lf from taskflow import retry from taskflow import test @@ -95,24 +94,6 @@ class LinearFlowTest(test.TestCase): (task1, task2, {'invariant': True}) ]) - def test_linear_flow_two_dependent_tasks_reverse_order(self): - task1 = _task(name='task1', provides=['a']) - task2 = _task(name='task2', requires=['a']) - f = lf.Flow('test') - self.assertRaises(exc.DependencyFailure, f.add, task2, task1) - - def test_linear_flow_two_dependent_tasks_reverse_order2(self): - task1 = _task(name='task1', provides=['a']) - task2 = _task(name='task2', requires=['a']) - f = lf.Flow('test').add(task2) - self.assertRaises(exc.DependencyFailure, f.add, task1) - - def test_linear_flow_two_task_same_provide(self): - task1 = _task(name='task1', provides=['a', 'b']) - task2 = _task(name='task2', provides=['a', 'c']) - f = lf.Flow('test') - self.assertRaises(exc.DependencyFailure, f.add, task2, task1) - def test_linear_flow_three_tasks(self): task1 = _task(name='task1') task2 = _task(name='task2') diff --git a/taskflow/tests/unit/test_flow_dependencies.py b/taskflow/tests/unit/test_flow_dependencies.py index 3ddb95d9..3f8e7509 100644 --- a/taskflow/tests/unit/test_flow_dependencies.py +++ b/taskflow/tests/unit/test_flow_dependencies.py @@ -186,13 +186,6 @@ class FlowDependenciesTest(test.TestCase): self.assertEqual(flow.requires, set(['a', 'b', 'c'])) self.assertEqual(flow.provides, set(['x', 'y', 'z', 'q'])) - def test_nested_flows_provides_same_values(self): - flow = lf.Flow('lf').add( - uf.Flow('uf').add(utils.TaskOneReturn(provides='x'))) - self.assertRaises(exceptions.DependencyFailure, - flow.add, - gf.Flow('gf').add(utils.TaskOneReturn(provides='x'))) - def test_graph_flow_requires_values(self): flow = gf.Flow('gf').add( utils.TaskOneArg('task1'), @@ -336,18 +329,6 @@ class FlowDependenciesTest(test.TestCase): self.assertEqual(flow.requires, set(['x', 'y', 'c'])) self.assertEqual(flow.provides, set(['a', 'b', 'z'])) - def test_linear_flow_retry_and_task_dependency_conflict(self): - flow = lf.Flow('lf', retry.AlwaysRevert('rt', requires=['x'])) - self.assertRaises(exceptions.DependencyFailure, - flow.add, - utils.TaskOneReturn(provides=['x'])) - - def test_linear_flow_retry_and_task_provide_same_value(self): - flow = lf.Flow('lf', retry.AlwaysRevert('rt', provides=['x'])) - self.assertRaises(exceptions.DependencyFailure, - flow.add, - utils.TaskOneReturn('t1', provides=['x'])) - def test_unordered_flow_retry_and_task(self): flow = uf.Flow('uf', retry.AlwaysRevert('rt', requires=['x', 'y'], @@ -399,21 +380,6 @@ class FlowDependenciesTest(test.TestCase): flow.add, utils.TaskOneReturn('t1', provides=['x'])) - def test_two_retries_provide_same_values_in_nested_flows(self): - flow = lf.Flow('lf', retry.AlwaysRevert('rt1', provides=['x'])) - self.assertRaises(exceptions.DependencyFailure, - flow.add, - lf.Flow('lf1', retry.AlwaysRevert('rt2', - provides=['x']))) - - def test_two_retries_provide_same_values(self): - flow = lf.Flow('lf').add( - lf.Flow('lf1', retry.AlwaysRevert('rt1', provides=['x']))) - self.assertRaises(exceptions.DependencyFailure, - flow.add, - lf.Flow('lf2', retry.AlwaysRevert('rt2', - provides=['x']))) - def test_builtin_retry_args(self): class FullArgsRetry(retry.AlwaysRevert): From 76641d86b89cdba23ac49d8c65011467a098f6dc Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 18 Jul 2014 12:58:18 -0700 Subject: [PATCH 014/240] Relax the unordered flow symbol constraints In order to make it possible to have a symbol tree we need to relax and remove the constraints that are being imposed by the unordered constraints and later move those constraint checks and validations into the engines compilation stage. Part of blueprint taskflow-improved-scoping Change-Id: I80718b4bc01fbf0dce6a95cd2fac7e6e2e1814d1 --- taskflow/patterns/unordered_flow.py | 49 +---------------- .../unit/patterns/test_unordered_flow.py | 42 +++++++------- taskflow/tests/unit/test_flow_dependencies.py | 55 +++++++++---------- 3 files changed, 51 insertions(+), 95 deletions(-) diff --git a/taskflow/patterns/unordered_flow.py b/taskflow/patterns/unordered_flow.py index b32fbc09..52bd286e 100644 --- a/taskflow/patterns/unordered_flow.py +++ b/taskflow/patterns/unordered_flow.py @@ -14,70 +14,25 @@ # License for the specific language governing permissions and limitations # under the License. -from taskflow import exceptions from taskflow import flow class Flow(flow.Flow): - """Unordered Flow pattern. + """Unordered flow pattern. A unordered (potentially nested) flow of *tasks/flows* that can be executed in any order as one unit and rolled back as one unit. - - NOTE(harlowja): Since the flow is unordered there can *not* be any - dependency between task/flow inputs (requirements) and - task/flow outputs (provided names/values). """ def __init__(self, name, retry=None): super(Flow, self).__init__(name, retry) # NOTE(imelnikov): A unordered flow is unordered, so we use # set instead of list to save children, children so that - # people using it don't depend on the ordering + # people using it don't depend on the ordering. self._children = set() def add(self, *items): """Adds a given task/tasks/flow/flows to this flow.""" - if not items: - return self - - # check that items don't provide anything that other - # part of flow provides or requires - provides = self.provides - old_requires = self.requires - for item in items: - item_provides = item.provides - bad_provs = item_provides & old_requires - if bad_provs: - raise exceptions.DependencyFailure( - "%(item)s provides %(oo)s that are required " - "by other item(s) of unordered flow %(flow)s" - % dict(item=item.name, flow=self.name, - oo=sorted(bad_provs))) - same_provides = provides & item.provides - if same_provides: - raise exceptions.DependencyFailure( - "%(item)s provides %(value)s but is already being" - " provided by %(flow)s and duplicate producers" - " are disallowed" - % dict(item=item.name, flow=self.name, - value=sorted(same_provides))) - provides |= item.provides - - # check that items don't require anything other children provides - if self.retry: - # NOTE(imelnikov): it is allowed to depend on value provided - # by retry controller of the flow - provides -= self.retry.provides - for item in items: - bad_reqs = provides & item.requires - if bad_reqs: - raise exceptions.DependencyFailure( - "%(item)s requires %(oo)s that are provided " - "by other item(s) of unordered flow %(flow)s" - % dict(item=item.name, flow=self.name, - oo=sorted(bad_reqs))) - self._children.update(items) return self diff --git a/taskflow/tests/unit/patterns/test_unordered_flow.py b/taskflow/tests/unit/patterns/test_unordered_flow.py index a4043fe2..e55cfad0 100644 --- a/taskflow/tests/unit/patterns/test_unordered_flow.py +++ b/taskflow/tests/unit/patterns/test_unordered_flow.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -from taskflow import exceptions as exc from taskflow.patterns import unordered_flow as uf from taskflow import retry from taskflow import test @@ -59,7 +58,7 @@ class UnorderedFlowTest(test.TestCase): self.assertEqual(f.requires, set(['a', 'b'])) self.assertEqual(f.provides, set(['c', 'd'])) - def test_unordered_flow_two_independent_tasks(self): + def test_unordered_flow_two_tasks(self): task1 = _task(name='task1') task2 = _task(name='task2') f = uf.Flow('test').add(task1, task2) @@ -68,35 +67,29 @@ class UnorderedFlowTest(test.TestCase): self.assertEqual(set(f), set([task1, task2])) self.assertEqual(list(f.iter_links()), []) - def test_unordered_flow_two_dependent_tasks(self): - task1 = _task(name='task1', provides=['a']) - task2 = _task(name='task2', requires=['a']) - f = uf.Flow('test') - self.assertRaises(exc.DependencyFailure, f.add, task1, task2) - - def test_unordered_flow_two_dependent_tasks_two_different_calls(self): + def test_unordered_flow_two_tasks_two_different_calls(self): task1 = _task(name='task1', provides=['a']) task2 = _task(name='task2', requires=['a']) f = uf.Flow('test').add(task1) - self.assertRaises(exc.DependencyFailure, f.add, task2) + f.add(task2) + self.assertEqual(len(f), 2) + self.assertEqual(set(['a']), f.requires) + self.assertEqual(set(['a']), f.provides) - def test_unordered_flow_two_dependent_tasks_reverse_order(self): + def test_unordered_flow_two_tasks_reverse_order(self): task1 = _task(name='task1', provides=['a']) task2 = _task(name='task2', requires=['a']) - f = uf.Flow('test') - self.assertRaises(exc.DependencyFailure, f.add, task2, task1) - - def test_unordered_flow_two_dependent_tasks_reverse_order2(self): - task1 = _task(name='task1', provides=['a']) - task2 = _task(name='task2', requires=['a']) - f = uf.Flow('test').add(task2) - self.assertRaises(exc.DependencyFailure, f.add, task1) + f = uf.Flow('test').add(task2).add(task1) + self.assertEqual(len(f), 2) + self.assertEqual(set(['a']), f.requires) + self.assertEqual(set(['a']), f.provides) def test_unordered_flow_two_task_same_provide(self): task1 = _task(name='task1', provides=['a', 'b']) task2 = _task(name='task2', provides=['a', 'c']) f = uf.Flow('test') - self.assertRaises(exc.DependencyFailure, f.add, task2, task1) + f.add(task2, task1) + self.assertEqual(len(f), 2) def test_unordered_flow_with_retry(self): ret = retry.AlwaysRevert(requires=['a'], provides=['b']) @@ -106,3 +99,12 @@ class UnorderedFlowTest(test.TestCase): self.assertEqual(f.requires, set(['a'])) self.assertEqual(f.provides, set(['b'])) + + def test_unordered_flow_with_retry_fully_satisfies(self): + ret = retry.AlwaysRevert(provides=['b', 'a']) + f = uf.Flow('test', ret) + f.add(_task(name='task1', requires=['a'])) + self.assertIs(f.retry, ret) + self.assertEqual(ret.name, 'test_retry') + self.assertEqual(f.requires, set([])) + self.assertEqual(f.provides, set(['b', 'a'])) diff --git a/taskflow/tests/unit/test_flow_dependencies.py b/taskflow/tests/unit/test_flow_dependencies.py index 3f8e7509..47604e81 100644 --- a/taskflow/tests/unit/test_flow_dependencies.py +++ b/taskflow/tests/unit/test_flow_dependencies.py @@ -130,24 +130,27 @@ class FlowDependenciesTest(test.TestCase): def test_unordered_flow_provides_required_values(self): flow = uf.Flow('uf') - self.assertRaises(exceptions.DependencyFailure, - flow.add, - utils.TaskOneReturn('task1', provides='x'), - utils.TaskOneArg('task2')) + flow.add(utils.TaskOneReturn('task1', provides='x'), + utils.TaskOneArg('task2')) + flow.add(utils.TaskOneReturn('task1', provides='x'), + utils.TaskOneArg('task2')) + self.assertEqual(set(['x']), flow.provides) + self.assertEqual(set(['x']), flow.requires) def test_unordered_flow_requires_provided_value_other_call(self): flow = uf.Flow('uf') flow.add(utils.TaskOneReturn('task1', provides='x')) - self.assertRaises(exceptions.DependencyFailure, - flow.add, - utils.TaskOneArg('task2')) + flow.add(utils.TaskOneArg('task2')) + self.assertEqual(set(['x']), flow.provides) + self.assertEqual(set(['x']), flow.requires) def test_unordered_flow_provides_required_value_other_call(self): flow = uf.Flow('uf') flow.add(utils.TaskOneArg('task2')) - self.assertRaises(exceptions.DependencyFailure, - flow.add, - utils.TaskOneReturn('task1', provides='x')) + flow.add(utils.TaskOneReturn('task1', provides='x')) + self.assertEqual(2, len(flow)) + self.assertEqual(set(['x']), flow.provides) + self.assertEqual(set(['x']), flow.requires) def test_unordered_flow_multi_provides_and_requires_values(self): flow = uf.Flow('uf').add( @@ -161,16 +164,14 @@ class FlowDependenciesTest(test.TestCase): def test_unordered_flow_provides_same_values(self): flow = uf.Flow('uf').add(utils.TaskOneReturn(provides='x')) - self.assertRaises(exceptions.DependencyFailure, - flow.add, - utils.TaskOneReturn(provides='x')) + flow.add(utils.TaskOneReturn(provides='x')) + self.assertEqual(set(['x']), flow.provides) def test_unordered_flow_provides_same_values_one_add(self): flow = uf.Flow('uf') - self.assertRaises(exceptions.DependencyFailure, - flow.add, - utils.TaskOneReturn(provides='x'), - utils.TaskOneReturn(provides='x')) + flow.add(utils.TaskOneReturn(provides='x'), + utils.TaskOneReturn(provides='x')) + self.assertEqual(set(['x']), flow.provides) def test_nested_flows_requirements(self): flow = uf.Flow('uf').add( @@ -339,24 +340,22 @@ class FlowDependenciesTest(test.TestCase): self.assertEqual(flow.requires, set(['x', 'y', 'c'])) self.assertEqual(flow.provides, set(['a', 'b', 'z'])) - def test_unordered_flow_retry_and_task_dependency_conflict(self): + def test_unordered_flow_retry_and_task_same_requires_provides(self): flow = uf.Flow('uf', retry.AlwaysRevert('rt', requires=['x'])) - self.assertRaises(exceptions.DependencyFailure, - flow.add, - utils.TaskOneReturn(provides=['x'])) + flow.add(utils.TaskOneReturn(provides=['x'])) + self.assertEqual(set(['x']), flow.requires) + self.assertEqual(set(['x']), flow.provides) def test_unordered_flow_retry_and_task_provide_same_value(self): flow = uf.Flow('uf', retry.AlwaysRevert('rt', provides=['x'])) - self.assertRaises(exceptions.DependencyFailure, - flow.add, - utils.TaskOneReturn('t1', provides=['x'])) + flow.add(utils.TaskOneReturn('t1', provides=['x'])) + self.assertEqual(set(['x']), flow.provides) def test_unordered_flow_retry_two_tasks_provide_same_value(self): flow = uf.Flow('uf', retry.AlwaysRevert('rt', provides=['y'])) - self.assertRaises(exceptions.DependencyFailure, - flow.add, - utils.TaskOneReturn('t1', provides=['x']), - utils.TaskOneReturn('t2', provides=['x'])) + flow.add(utils.TaskOneReturn('t1', provides=['x']), + utils.TaskOneReturn('t2', provides=['x'])) + self.assertEqual(set(['x', 'y']), flow.provides) def test_graph_flow_retry_and_task(self): flow = gf.Flow('gf', retry.AlwaysRevert('rt', From d6ef68762e847373be0584820fa0557fcbd5003f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 18 Jul 2014 12:58:18 -0700 Subject: [PATCH 015/240] Relax the graph flow symbol constraints In order to make it possible to have a symbol tree we need to relax the constraints that are being imposed by the graph flow. Part of blueprint taskflow-improved-scoping Change-Id: I2e14de2131c3ba4e3e4eb3108477583d0f02dae2 --- taskflow/exceptions.py | 4 + taskflow/patterns/graph_flow.py | 150 +++++++++++------- .../tests/unit/patterns/test_graph_flow.py | 35 +++- taskflow/tests/unit/test_flow_dependencies.py | 18 +-- 4 files changed, 140 insertions(+), 67 deletions(-) diff --git a/taskflow/exceptions.py b/taskflow/exceptions.py index e5b9a9c2..312b931c 100644 --- a/taskflow/exceptions.py +++ b/taskflow/exceptions.py @@ -98,6 +98,10 @@ class DependencyFailure(TaskFlowException): """Raised when some type of dependency problem occurs.""" +class AmbiguousDependency(DependencyFailure): + """Raised when some type of ambiguous dependency problem occurs.""" + + class MissingDependencies(DependencyFailure): """Raised when a entity has dependencies that can not be satisfied.""" MESSAGE_TPL = ("%(who)s requires %(requirements)s but no other entity" diff --git a/taskflow/patterns/graph_flow.py b/taskflow/patterns/graph_flow.py index 658e051c..f07e7435 100644 --- a/taskflow/patterns/graph_flow.py +++ b/taskflow/patterns/graph_flow.py @@ -21,6 +21,22 @@ from taskflow import flow from taskflow.types import graph as gr +def _unsatisfied_requires(node, graph, *additional_provided): + """Extracts the unsatisified symbol requirements of a single node.""" + requires = set(node.requires) + if not requires: + return requires + for provided in additional_provided: + requires = requires - provided + if not requires: + return requires + for pred in graph.bfs_predecessors_iter(node): + requires = requires - pred.provides + if not requires: + return requires + return requires + + class Flow(flow.Flow): """Graph flow pattern. @@ -80,33 +96,59 @@ class Flow(flow.Flow): if not graph.is_directed_acyclic(): raise exc.DependencyFailure("No path through the items in the" " graph produces an ordering that" - " will allow for correct dependency" - " resolution") - self._graph = graph - self._graph.freeze() + " will allow for logical" + " edge traversal") + self._graph = graph.freeze() - def add(self, *items): - """Adds a given task/tasks/flow/flows to this flow.""" + def add(self, *items, **kwargs): + """Adds a given task/tasks/flow/flows to this flow. + + :param items: items to add to the flow + :param kwargs: keyword arguments, the two keyword arguments + currently processed are: + + * ``resolve_requires`` a boolean that when true (the + default) implies that when items are added their + symbol requirements will be matched to existing items + and links will be automatically made to those + providers. If multiple possible providers exist + then a AmbiguousDependency exception will be raised. + * ``resolve_existing``, a boolean that when true (the + default) implies that on addition of a new item that + existing items will have their requirements scanned + for symbols that this newly added item can provide. + If a match is found a link is automatically created + from the newly added item to the requiree. + """ items = [i for i in items if not self._graph.has_node(i)] if not items: return self - requirements = collections.defaultdict(list) - provided = {} + # This syntax will *hopefully* be better in future versions of python. + # + # See: http://legacy.python.org/dev/peps/pep-3102/ (python 3.0+) + resolve_requires = bool(kwargs.get('resolve_requires', True)) + resolve_existing = bool(kwargs.get('resolve_existing', True)) - def update_requirements(node): - for value in node.requires: - requirements[value].append(node) + # Figure out what the existing nodes *still* require and what they + # provide so we can do this lookup later when inferring. + required = collections.defaultdict(list) + provided = collections.defaultdict(list) - for node in self: - update_requirements(node) - for value in node.provides: - provided[value] = node + retry_provides = set() + if self._retry is not None: + for value in self._retry.requires: + required[value].append(self._retry) + for value in self._retry.provides: + retry_provides.add(value) + provided[value].append(self._retry) - if self.retry: - update_requirements(self.retry) - provided.update(dict((k, self.retry) - for k in self.retry.provides)) + for item in self._graph.nodes_iter(): + for value in _unsatisfied_requires(item, self._graph, + retry_provides): + required[value].append(item) + for value in item.provides: + provided[value].append(item) # NOTE(harlowja): Add items and edges to a temporary copy of the # underlying graph and only if that is successful added to do we then @@ -114,37 +156,41 @@ class Flow(flow.Flow): tmp_graph = gr.DiGraph(self._graph) for item in items: tmp_graph.add_node(item) - update_requirements(item) - for value in item.provides: - if value in provided: - raise exc.DependencyFailure( - "%(item)s provides %(value)s but is already being" - " provided by %(flow)s and duplicate producers" - " are disallowed" - % dict(item=item.name, - flow=provided[value].name, - value=value)) - if self.retry and value in self.retry.requires: - raise exc.DependencyFailure( - "Flows retry controller %(retry)s requires %(value)s " - "but item %(item)s being added to the flow produces " - "that item, this creates a cyclic dependency and is " - "disallowed" - % dict(item=item.name, - retry=self.retry.name, - value=value)) - provided[value] = item - for value in item.requires: - if value in provided: - self._link(provided[value], item, - graph=tmp_graph, reason=value) + # Try to find a valid provider. + if resolve_requires: + for value in _unsatisfied_requires(item, tmp_graph, + retry_provides): + if value in provided: + providers = provided[value] + if len(providers) > 1: + provider_names = [n.name for n in providers] + raise exc.AmbiguousDependency( + "Resolution error detected when" + " adding %(item)s, multiple" + " providers %(providers)s found for" + " required symbol '%(value)s'" + % dict(item=item.name, + providers=sorted(provider_names), + value=value)) + else: + self._link(providers[0], item, + graph=tmp_graph, reason=value) + else: + required[value].append(item) for value in item.provides: - if value in requirements: - for node in requirements[value]: - self._link(item, node, - graph=tmp_graph, reason=value) + provided[value].append(item) + + # See if what we provide fulfills any existing requiree. + if resolve_existing: + for value in item.provides: + if value in required: + for requiree in list(required[value]): + if requiree is not item: + self._link(item, requiree, + graph=tmp_graph, reason=value) + required[value].remove(requiree) self._swap(tmp_graph) return self @@ -177,15 +223,7 @@ class Flow(flow.Flow): retry_provides.update(self._retry.provides) g = self._get_subgraph() for item in g.nodes_iter(): - item_requires = item.requires - retry_provides - # Now scan predecessors to see if they provide what we want. - if item_requires: - for pred_item in g.bfs_predecessors_iter(item): - item_requires = item_requires - pred_item.provides - if not item_requires: - break - if item_requires: - requires.update(item_requires) + requires.update(_unsatisfied_requires(item, g, retry_provides)) return frozenset(requires) diff --git a/taskflow/tests/unit/patterns/test_graph_flow.py b/taskflow/tests/unit/patterns/test_graph_flow.py index c7dad38e..62dbc287 100644 --- a/taskflow/tests/unit/patterns/test_graph_flow.py +++ b/taskflow/tests/unit/patterns/test_graph_flow.py @@ -97,7 +97,40 @@ class GraphFlowTest(test.TestCase): task1 = _task(name='task1', provides=['a', 'b']) task2 = _task(name='task2', provides=['a', 'c']) f = gf.Flow('test') - self.assertRaises(exc.DependencyFailure, f.add, task2, task1) + f.add(task2, task1) + self.assertEqual(set(['a', 'b', 'c']), f.provides) + + def test_graph_flow_ambiguous_provides(self): + task1 = _task(name='task1', provides=['a', 'b']) + task2 = _task(name='task2', provides=['a']) + f = gf.Flow('test') + f.add(task1, task2) + self.assertEqual(set(['a', 'b']), f.provides) + task3 = _task(name='task3', requires=['a']) + self.assertRaises(exc.AmbiguousDependency, f.add, task3) + + def test_graph_flow_no_resolve_requires(self): + task1 = _task(name='task1', provides=['a', 'b', 'c']) + task2 = _task(name='task2', requires=['a', 'b']) + f = gf.Flow('test') + f.add(task1, task2, resolve_requires=False) + self.assertEqual(set(['a', 'b']), f.requires) + + def test_graph_flow_no_resolve_existing(self): + task1 = _task(name='task1', requires=['a', 'b']) + task2 = _task(name='task2', provides=['a', 'b']) + f = gf.Flow('test') + f.add(task1) + f.add(task2, resolve_existing=False) + self.assertEqual(set(['a', 'b']), f.requires) + + def test_graph_flow_resolve_existing(self): + task1 = _task(name='task1', requires=['a', 'b']) + task2 = _task(name='task2', provides=['a', 'b']) + f = gf.Flow('test') + f.add(task1) + f.add(task2, resolve_existing=True) + self.assertEqual(set([]), f.requires) def test_graph_flow_with_retry(self): ret = retry.AlwaysRevert(requires=['a'], provides=['b']) diff --git a/taskflow/tests/unit/test_flow_dependencies.py b/taskflow/tests/unit/test_flow_dependencies.py index 47604e81..69f4a8fe 100644 --- a/taskflow/tests/unit/test_flow_dependencies.py +++ b/taskflow/tests/unit/test_flow_dependencies.py @@ -218,9 +218,8 @@ class FlowDependenciesTest(test.TestCase): def test_graph_flow_provides_provided_value_other_call(self): flow = gf.Flow('gf') flow.add(utils.TaskOneReturn('task1', provides='x')) - self.assertRaises(exceptions.DependencyFailure, - flow.add, - utils.TaskOneReturn('task2', provides='x')) + flow.add(utils.TaskOneReturn('task2', provides='x')) + self.assertEqual(set(['x']), flow.provides) def test_graph_flow_multi_provides_and_requires_values(self): flow = gf.Flow('gf').add( @@ -367,17 +366,16 @@ class FlowDependenciesTest(test.TestCase): self.assertEqual(flow.requires, set(['x', 'y', 'c'])) self.assertEqual(flow.provides, set(['a', 'b', 'z'])) - def test_graph_flow_retry_and_task_dependency_conflict(self): + def test_graph_flow_retry_and_task_dependency_provide_require(self): flow = gf.Flow('gf', retry.AlwaysRevert('rt', requires=['x'])) - self.assertRaises(exceptions.DependencyFailure, - flow.add, - utils.TaskOneReturn(provides=['x'])) + flow.add(utils.TaskOneReturn(provides=['x'])) + self.assertEqual(set(['x']), flow.provides) + self.assertEqual(set(['x']), flow.requires) def test_graph_flow_retry_and_task_provide_same_value(self): flow = gf.Flow('gf', retry.AlwaysRevert('rt', provides=['x'])) - self.assertRaises(exceptions.DependencyFailure, - flow.add, - utils.TaskOneReturn('t1', provides=['x'])) + flow.add(utils.TaskOneReturn('t1', provides=['x'])) + self.assertEqual(set(['x']), flow.provides) def test_builtin_retry_args(self): From d98f23d9c2b7e79f02dabfc04fbe90cd7e03545c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 28 Jul 2014 10:51:26 -0700 Subject: [PATCH 016/240] Add a couple of scope shadowing test cases Since we now support symbol name shadowing we should add a few test cases that ensure that the correct shadowing order/capability is working as expected. Change-Id: I95bad23a33304ddbfa8715a00c913891d4c51f5d --- .../tests/unit/test_action_engine_scoping.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/taskflow/tests/unit/test_action_engine_scoping.py b/taskflow/tests/unit/test_action_engine_scoping.py index e2de763f..b4429264 100644 --- a/taskflow/tests/unit/test_action_engine_scoping.py +++ b/taskflow/tests/unit/test_action_engine_scoping.py @@ -246,3 +246,52 @@ class MixedPatternScopingTest(test.TestCase): break self.assertGreater(first_subroot, first_root) self.assertEqual(scope[0][-2:], ['root.2', 'root.1']) + + def test_shadow_graph(self): + r = gf.Flow("root") + customer = test_utils.ProvidesRequiresTask("customer", + provides=['dog'], + requires=[]) + customer2 = test_utils.ProvidesRequiresTask("customer2", + provides=['dog'], + requires=[]) + washer = test_utils.ProvidesRequiresTask("washer", + requires=['dog'], + provides=['wash']) + r.add(customer, washer) + r.add(customer2, resolve_requires=False) + r.link(customer2, washer) + + c = compiler.PatternCompiler(r).compile() + + # The order currently is *not* guaranteed to be 'customer' before + # 'customer2' or the reverse, since either can occur before the + # washer; since *either* is a valid topological ordering of the + # dependencies... + # + # This may be different after/if the following is resolved: + # + # https://github.com/networkx/networkx/issues/1181 (and a few others) + self.assertEqual(set(['customer', 'customer2']), + set(_get_scopes(c, washer)[0])) + self.assertEqual([], _get_scopes(c, customer2)) + self.assertEqual([], _get_scopes(c, customer)) + + def test_shadow_linear(self): + r = lf.Flow("root") + + customer = test_utils.ProvidesRequiresTask("customer", + provides=['dog'], + requires=[]) + customer2 = test_utils.ProvidesRequiresTask("customer2", + provides=['dog'], + requires=[]) + washer = test_utils.ProvidesRequiresTask("washer", + requires=['dog'], + provides=['wash']) + r.add(customer, customer2, washer) + + c = compiler.PatternCompiler(r).compile() + + # This order is guaranteed... + self.assertEqual(['customer2', 'customer'], _get_scopes(c, washer)[0]) From 3465e0340b6c461dc3cac25321e7a13cca37b8e8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 3 Sep 2014 18:38:02 -0700 Subject: [PATCH 017/240] Reduce unused tox environments Since the openstack-ci only tests 5 of the many tox enviroments we had listed and from my knowledge nobody else is testing using those other non-standard enviroments just reduce the set of applicable environments to the ones that are being used (which also means we can remove the usage of toxgen and just stick with a simpler file that is easier to use, modify and adjust). Change-Id: I4d2302594b9f9e8741f9693cb358efc1418bd45d --- README.rst | 14 +--- tox-tmpl.ini | 113 --------------------------- tox.ini | 210 +++------------------------------------------------ 3 files changed, 12 insertions(+), 325 deletions(-) delete mode 100644 tox-tmpl.ini diff --git a/README.rst b/README.rst index 24c6e5d9..dea1f4ef 100644 --- a/README.rst +++ b/README.rst @@ -34,16 +34,8 @@ Tox.ini Our ``tox.ini`` file describes several test environments that allow to test TaskFlow with different python versions and sets of requirements installed. - -To generate the ``tox.ini`` file, use the ``toxgen.py`` script by first -installing `toxgen`_ and then provide that script as input the ``tox-tmpl.ini`` -file to generate the final ``tox.ini`` file. - -*For example:* - -:: - - $ toxgen.py -i tox-tmpl.ini -o tox.ini +Please refer to the `tox`_ documentation to understand how to make these test +environments work for you. Developer documentation ----------------------- @@ -56,5 +48,5 @@ We also have sphinx documentation in ``docs/source``. $ python setup.py build_sphinx -.. _toxgen: https://pypi.python.org/pypi/toxgen/ +.. _tox: http://testrun.org/tox/latest/ .. _developer documentation: http://docs.openstack.org/developer/taskflow/ diff --git a/tox-tmpl.ini b/tox-tmpl.ini deleted file mode 100644 index 8ad9f332..00000000 --- a/tox-tmpl.ini +++ /dev/null @@ -1,113 +0,0 @@ -# NOTE(harlowja): this is a template, not a fully-generated tox.ini, use toxgen -# to translate this into a fully specified tox.ini file before using. Changes -# made to tox.ini will only be reflected if ran through the toxgen generator. - -[tox] -minversion = 1.6 -skipsdist = True - -[testenv] -usedevelop = True -install_command = pip install {opts} {packages} -setenv = VIRTUAL_ENV={envdir} -deps = -r{toxinidir}/test-requirements.txt - alembic>=0.4.1 - psycopg2 - kazoo>=1.3.1 - kombu>=2.4.8 -commands = python setup.py testr --slowest --testr-args='{posargs}' - -[tox:jenkins] -downloadcache = ~/cache/pip - -[testenv:pep8] -commands = flake8 {posargs} - -[testenv:pylint] -setenv = VIRTUAL_ENV={envdir} -deps = -r{toxinidir}/requirements-py2.txt - pylint==0.26.0 -commands = pylint --rcfile=pylintrc taskflow - -[testenv:cover] -basepython = python2.7 -deps = {[testenv:py27]deps} -commands = python setup.py testr --coverage --testr-args='{posargs}' - -[testenv:venv] -commands = {posargs} - -[flake8] -# H904 Wrap long lines in parentheses instead of a backslash -ignore = H904 -builtins = _ -exclude = .venv,.tox,dist,doc,./taskflow/openstack/common,*egg,.git,build,tools - -# NOTE(imelnikov): pyXY envs are considered to be default, so they must have -# richest set of test requirements -[testenv:py26] -deps = {[testenv:py26-sa7-mysql-ev]deps} - -[testenv:py27] -deps = -r{toxinidir}/requirements-py2.txt - -r{toxinidir}/optional-requirements.txt - -r{toxinidir}/test-requirements.txt - doc8>=0.3.4 -commands = - python setup.py testr --slowest --testr-args='{posargs}' - sphinx-build -b doctest doc/source doc/build - doc8 doc/source - -[testenv:py33] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py3.txt - SQLAlchemy>=0.7.8,<=0.9.99 - -# NOTE(imelnikov): psycopg2 is not supported on pypy -[testenv:pypy] -deps = -r{toxinidir}/requirements-py2.txt - -r{toxinidir}/test-requirements.txt - SQLAlchemy>=0.7.8,<=0.9.99 - alembic>=0.4.1 - kazoo>=1.3.1 - kombu>=2.4.8 - -[axes] -python = py26,py27 -sqlalchemy = sa7,sa8,sa9 -mysql = mysql,pymysql -eventlet = ev,* - -[axis:python:py26] -basepython = python2.6 -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - -[axis:python:py27] -basepython = python2.7 -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - -[axis:eventlet:ev] -deps = - eventlet>=0.13.0 - -[axis:sqlalchemy:sa7] -deps = - SQLAlchemy>=0.7.8,<=0.7.99 - -[axis:sqlalchemy:sa8] -deps = - SQLAlchemy>=0.8,<=0.8.99 - -[axis:sqlalchemy:sa9] -deps = - SQLAlchemy>=0.9,<=0.9.99 - -[axis:mysql:mysql] -deps = - MySQL-python - -[axis:mysql:pymysql] -deps = - pyMySQL diff --git a/tox.ini b/tox.ini index 0283c14d..efc7bc3b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,3 @@ -# DO NOT EDIT THIS FILE - it is machine generated from tox-tmpl.ini - [tox] minversion = 1.6 skipsdist = True @@ -7,33 +5,10 @@ envlist = cover, pep8, py26, py26-sa7-mysql, - py26-sa7-mysql-ev, - py26-sa7-pymysql, - py26-sa7-pymysql-ev, - py26-sa8-mysql, - py26-sa8-mysql-ev, - py26-sa8-pymysql, - py26-sa8-pymysql-ev, - py26-sa9-mysql, - py26-sa9-mysql-ev, - py26-sa9-pymysql, - py26-sa9-pymysql-ev, py27, - py27-sa7-mysql, - py27-sa7-mysql-ev, - py27-sa7-pymysql, - py27-sa7-pymysql-ev, py27-sa8-mysql, - py27-sa8-mysql-ev, - py27-sa8-pymysql, - py27-sa8-pymysql-ev, - py27-sa9-mysql, - py27-sa9-mysql-ev, - py27-sa9-pymysql, - py27-sa9-pymysql-ev, py33, pylint, - pypy [testenv] usedevelop = True @@ -67,12 +42,20 @@ commands = python setup.py testr --coverage --testr-args='{posargs}' commands = {posargs} [flake8] +# H904 Wrap long lines in parentheses instead of a backslash ignore = H904 builtins = _ exclude = .venv,.tox,dist,doc,./taskflow/openstack/common,*egg,.git,build,tools +# NOTE(imelnikov): pyXY envs are considered to be default, so they must have +# richest set of test requirements [testenv:py26] -deps = {[testenv:py26-sa7-mysql-ev]deps} +deps = {[testenv]deps} + -r{toxinidir}/requirements-py2.txt + SQLAlchemy>=0.7.8,<=0.7.99 + MySQL-python + eventlet>=0.13.0 +basepython = python2.6 [testenv:py27] deps = -r{toxinidir}/requirements-py2.txt @@ -89,22 +72,6 @@ deps = {[testenv]deps} -r{toxinidir}/requirements-py3.txt SQLAlchemy>=0.7.8,<=0.9.99 -[testenv:pypy] -deps = -r{toxinidir}/requirements-py2.txt - -r{toxinidir}/test-requirements.txt - SQLAlchemy>=0.7.8,<=0.9.99 - alembic>=0.4.1 - kazoo>=1.3.1 - kombu>=2.4.8 - -[testenv:py26-sa7-mysql-ev] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.7.8,<=0.7.99 - MySQL-python - eventlet>=0.13.0 -basepython = python2.6 - [testenv:py26-sa7-mysql] deps = {[testenv]deps} -r{toxinidir}/requirements-py2.txt @@ -112,168 +79,9 @@ deps = {[testenv]deps} MySQL-python basepython = python2.6 -[testenv:py26-sa7-pymysql-ev] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.7.8,<=0.7.99 - pyMySQL - eventlet>=0.13.0 -basepython = python2.6 - -[testenv:py26-sa7-pymysql] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.7.8,<=0.7.99 - pyMySQL -basepython = python2.6 - -[testenv:py26-sa8-mysql-ev] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.8,<=0.8.99 - MySQL-python - eventlet>=0.13.0 -basepython = python2.6 - -[testenv:py26-sa8-mysql] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.8,<=0.8.99 - MySQL-python -basepython = python2.6 - -[testenv:py26-sa8-pymysql-ev] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.8,<=0.8.99 - pyMySQL - eventlet>=0.13.0 -basepython = python2.6 - -[testenv:py26-sa8-pymysql] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.8,<=0.8.99 - pyMySQL -basepython = python2.6 - -[testenv:py26-sa9-mysql-ev] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.9,<=0.9.99 - MySQL-python - eventlet>=0.13.0 -basepython = python2.6 - -[testenv:py26-sa9-mysql] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.9,<=0.9.99 - MySQL-python -basepython = python2.6 - -[testenv:py26-sa9-pymysql-ev] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.9,<=0.9.99 - pyMySQL - eventlet>=0.13.0 -basepython = python2.6 - -[testenv:py26-sa9-pymysql] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.9,<=0.9.99 - pyMySQL -basepython = python2.6 - -[testenv:py27-sa7-mysql-ev] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.7.8,<=0.7.99 - MySQL-python - eventlet>=0.13.0 -basepython = python2.7 - -[testenv:py27-sa7-mysql] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.7.8,<=0.7.99 - MySQL-python -basepython = python2.7 - -[testenv:py27-sa7-pymysql-ev] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.7.8,<=0.7.99 - pyMySQL - eventlet>=0.13.0 -basepython = python2.7 - -[testenv:py27-sa7-pymysql] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.7.8,<=0.7.99 - pyMySQL -basepython = python2.7 - -[testenv:py27-sa8-mysql-ev] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.8,<=0.8.99 - MySQL-python - eventlet>=0.13.0 -basepython = python2.7 - [testenv:py27-sa8-mysql] deps = {[testenv]deps} -r{toxinidir}/requirements-py2.txt SQLAlchemy>=0.8,<=0.8.99 MySQL-python basepython = python2.7 - -[testenv:py27-sa8-pymysql-ev] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.8,<=0.8.99 - pyMySQL - eventlet>=0.13.0 -basepython = python2.7 - -[testenv:py27-sa8-pymysql] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.8,<=0.8.99 - pyMySQL -basepython = python2.7 - -[testenv:py27-sa9-mysql-ev] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.9,<=0.9.99 - MySQL-python - eventlet>=0.13.0 -basepython = python2.7 - -[testenv:py27-sa9-mysql] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.9,<=0.9.99 - MySQL-python -basepython = python2.7 - -[testenv:py27-sa9-pymysql-ev] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.9,<=0.9.99 - pyMySQL - eventlet>=0.13.0 -basepython = python2.7 - -[testenv:py27-sa9-pymysql] -deps = {[testenv]deps} - -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.9,<=0.9.99 - pyMySQL -basepython = python2.7 - From 6ffb74c7a6696d880543fe1f5011efe8e4d804f6 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 8 Sep 2014 17:14:57 -0700 Subject: [PATCH 018/240] Add a docs virtualenv The various projects are standardizing on a docs venv which will eventually be called by the infra project to build the docs for a project. To enable our usage of this environment add a section so that it can be called upon. Change-Id: I420eb0d6e6f9f6f24bc493d4342478daba27ad1e --- tox.ini | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tox.ini b/tox.ini index efc7bc3b..f8717289 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,7 @@ minversion = 1.6 skipsdist = True envlist = cover, + docs, pep8, py26, py26-sa7-mysql, @@ -21,6 +22,15 @@ deps = -r{toxinidir}/test-requirements.txt kombu>=2.4.8 commands = python setup.py testr --slowest --testr-args='{posargs}' +[testenv:docs] +basepython = python2.7 +deps = -r{toxinidir}/requirements-py2.txt + -r{toxinidir}/test-requirements.txt + -r{toxinidir}/optional-requirements.txt + doc8 +commands = python setup.py build_sphinx + doc8 doc/source + [tox:jenkins] downloadcache = ~/cache/pip From 155326b6e7066f01bb82d02d184c3c2069159b45 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 8 Sep 2014 14:57:49 -0700 Subject: [PATCH 019/240] Update the state graph builder to use state machine type To build the state diagrams switch from using a custom algorithm that builds a state machine in a table to using our state machine class to build those same machines before translation into a dot diagram. This also allows us to use the state machine that already is being used in the action engine as the source of transitions to build the dot diagram with. Change-Id: I4a0d16a3fb7c620c2774b535ab952b5d5006e9e9 --- doc/source/img/engine_states.svg | 6 +- doc/source/img/flow_states.svg | 6 +- doc/source/img/retry_states.svg | 6 +- doc/source/img/task_states.svg | 6 +- doc/source/states.rst | 6 ++ tools/state_graph.py | 122 ++++++++++++++++++------------- 6 files changed, 91 insertions(+), 61 deletions(-) diff --git a/doc/source/img/engine_states.svg b/doc/source/img/engine_states.svg index 497c31ef..8ef68c3e 100644 --- a/doc/source/img/engine_states.svg +++ b/doc/source/img/engine_states.svg @@ -1,8 +1,8 @@ - - -Engines statesRESUMINGSCHEDULINGWAITINGSUCCESSSUSPENDEDREVERTEDANALYZINGstart + +Engines statesGAME_OVERREVERTEDSUCCESSSUSPENDEDFAILUREUNDEFINEDRESUMINGSCHEDULINGANALYZINGWAITINGstart diff --git a/doc/source/img/flow_states.svg b/doc/source/img/flow_states.svg index c6d9825e..5a1cdcbd 100644 --- a/doc/source/img/flow_states.svg +++ b/doc/source/img/flow_states.svg @@ -1,8 +1,8 @@ - - -Flow statesPENDINGRUNNINGRESUMINGFAILURESUCCESSREVERTEDSUSPENDINGSUSPENDEDstart + +Flow statesPENDINGRUNNINGFAILURESUSPENDINGREVERTEDSUCCESSRESUMINGSUSPENDEDstart diff --git a/doc/source/img/retry_states.svg b/doc/source/img/retry_states.svg index 014516e0..a2ba2fa9 100644 --- a/doc/source/img/retry_states.svg +++ b/doc/source/img/retry_states.svg @@ -1,8 +1,8 @@ - - -Retries statesPENDINGRUNNINGFAILURESUCCESSREVERTINGRETRYINGREVERTEDstart + +Retries statesPENDINGRUNNINGSUCCESSFAILURERETRYINGREVERTINGREVERTEDstart diff --git a/doc/source/img/task_states.svg b/doc/source/img/task_states.svg index f40501ac..c281be8e 100644 --- a/doc/source/img/task_states.svg +++ b/doc/source/img/task_states.svg @@ -1,8 +1,8 @@ - - -Tasks statesPENDINGRUNNINGFAILURESUCCESSREVERTINGREVERTEDstart + +Tasks statesPENDINGRUNNINGSUCCESSFAILUREREVERTINGREVERTEDstart diff --git a/doc/source/states.rst b/doc/source/states.rst index 02fcaf15..432e0079 100644 --- a/doc/source/states.rst +++ b/doc/source/states.rst @@ -22,11 +22,17 @@ Engine **SUCCESS** - Completed successfully. +**FAILURE** - Completed unsuccessfully. + **REVERTED** - Reverting was induced and all atoms were **not** completed successfully. **SUSPENDED** - Suspended while running. +**UNDEFINED** - *Internal state.* + +**GAME_OVER** - *Internal state.* + Flow ==== diff --git a/tools/state_graph.py b/tools/state_graph.py index 77b85636..e83426bf 100755 --- a/tools/state_graph.py +++ b/tools/state_graph.py @@ -1,5 +1,19 @@ #!/usr/bin/env python +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + import optparse import os import sys @@ -8,13 +22,39 @@ top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) sys.path.insert(0, top_dir) -import networkx as nx - # To get this installed you may have to follow: # https://code.google.com/p/pydot/issues/detail?id=93 (until fixed). import pydot +from taskflow.engines.action_engine import runner from taskflow import states +from taskflow.types import fsm + + +# This is just needed to get at the runner builder object (we will not +# actually be running it...). +class DummyRuntime(object): + def __init__(self): + self.analyzer = None + self.completer = None + self.scheduler = None + self.storage = None + + +def make_machine(start_state, transitions, disallowed): + machine = fsm.FSM(start_state) + machine.add_state(start_state) + for (start_state, end_state) in transitions: + if start_state in disallowed or end_state in disallowed: + continue + if start_state not in machine: + machine.add_state(start_state) + if end_state not in machine: + machine.add_state(end_state) + # Make a fake event (not used anyway)... + event = "on_%s" % (end_state) + machine.add_transition(start_state, end_state, event.lower()) + return machine def main(): @@ -45,77 +85,61 @@ def main(): if sum([int(i) for i in types]) > 1: parser.error("Only one of task/retry/engines may be specified.") - disallowed = set() - start_node = states.PENDING + internal_states = list() + ordering = 'in' if options.tasks: - source = list(states._ALLOWED_TASK_TRANSITIONS) source_type = "Tasks" - disallowed.add(states.RETRYING) + source = make_machine(states.PENDING, + list(states._ALLOWED_TASK_TRANSITIONS), + [states.RETRYING]) elif options.retries: - source = list(states._ALLOWED_TASK_TRANSITIONS) source_type = "Retries" + source = make_machine(states.PENDING, + list(states._ALLOWED_TASK_TRANSITIONS), []) elif options.engines: - # TODO(harlowja): place this in states.py - source = [ - (states.RESUMING, states.SCHEDULING), - (states.SCHEDULING, states.WAITING), - (states.WAITING, states.ANALYZING), - (states.ANALYZING, states.SCHEDULING), - (states.ANALYZING, states.WAITING), - ] - for u in (states.SCHEDULING, states.ANALYZING): - for v in (states.SUSPENDED, states.SUCCESS, states.REVERTED): - source.append((u, v)) source_type = "Engines" - start_node = states.RESUMING + r = runner.Runner(DummyRuntime(), None) + source, memory = r.builder.build() + internal_states.extend(runner._META_STATES) + ordering = 'out' else: - source = list(states._ALLOWED_FLOW_TRANSITIONS) source_type = "Flow" - - transitions = nx.DiGraph() - for (u, v) in source: - if u not in disallowed: - transitions.add_node(u) - if v not in disallowed: - transitions.add_node(v) - for (u, v) in source: - if not transitions.has_node(u) or not transitions.has_node(v): - continue - transitions.add_edge(u, v) + source = make_machine(states.PENDING, + list(states._ALLOWED_FLOW_TRANSITIONS), []) graph_name = "%s states" % source_type g = pydot.Dot(graph_name=graph_name, rankdir='LR', nodesep='0.25', overlap='false', ranksep="0.5", size="11x8.5", - splines='true', ordering='in') + splines='true', ordering=ordering) node_attrs = { 'fontsize': '11', } nodes = {} - nodes_order = [] - edges_added = [] - for (u, v) in nx.bfs_edges(transitions, source=start_node): - if u not in nodes: - nodes[u] = pydot.Node(u, **node_attrs) - g.add_node(nodes[u]) - nodes_order.append(u) - if v not in nodes: - nodes[v] = pydot.Node(v, **node_attrs) - g.add_node(nodes[v]) - nodes_order.append(v) - for u in nodes_order: - for v in transitions.successors_iter(u): - if (u, v) not in edges_added: - g.add_edge(pydot.Edge(nodes[u], nodes[v])) - edges_added.append((u, v)) + for (start_state, _on_event, end_state) in source: + if start_state not in nodes: + start_node_attrs = node_attrs.copy() + if start_state in internal_states: + start_node_attrs['fontcolor'] = 'blue' + nodes[start_state] = pydot.Node(start_state, **start_node_attrs) + g.add_node(nodes[start_state]) + if end_state not in nodes: + end_node_attrs = node_attrs.copy() + if end_state in internal_states: + end_node_attrs['fontcolor'] = 'blue' + nodes[end_state] = pydot.Node(end_state, **end_node_attrs) + g.add_node(nodes[end_state]) + g.add_edge(pydot.Edge(nodes[start_state], nodes[end_state])) + start = pydot.Node("__start__", shape="point", width="0.1", xlabel='start', fontcolor='green', **node_attrs) g.add_node(start) - g.add_edge(pydot.Edge(start, nodes[start_node], style='dotted')) + g.add_edge(pydot.Edge(start, nodes[source.start_state], style='dotted')) print("*" * len(graph_name)) print(graph_name) print("*" * len(graph_name)) + print(source.pformat()) print(g.to_string().strip()) g.write(options.filename, format=options.format) From 26793dcbf9519b6ffd904415d7264bee2ac9b7fe Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 11 Sep 2014 16:17:18 -0700 Subject: [PATCH 020/240] Add a state machine copy() method In order to move the states.py constants to a state machine object we need to be able to copy that object that will be defined there so that it can be used by those states users. Change-Id: I5fc21370d91ff578bd21c74f4bd7b8c0e130b144 --- taskflow/tests/unit/test_types.py | 36 +++++++++++++++++++++++++++++++ taskflow/types/fsm.py | 14 ++++++++++++ 2 files changed, 50 insertions(+) diff --git a/taskflow/tests/unit/test_types.py b/taskflow/tests/unit/test_types.py index 141cdfc8..5382466e 100644 --- a/taskflow/tests/unit/test_types.py +++ b/taskflow/tests/unit/test_types.py @@ -292,6 +292,42 @@ class FSMTest(test.TestCase): self.assertRaises(fsm.NotInitialized, self.jumper.process_event, 'jump') + def test_copy_states(self): + c = fsm.FSM('down') + self.assertEqual(0, len(c.states)) + d = c.copy() + c.add_state('up') + c.add_state('down') + self.assertEqual(2, len(c.states)) + self.assertEqual(0, len(d.states)) + + def test_copy_reactions(self): + c = fsm.FSM('down') + d = c.copy() + + c.add_state('down') + c.add_state('up') + c.add_reaction('down', 'jump', lambda *args: 'up') + c.add_transition('down', 'up', 'jump') + + self.assertEqual(1, c.events) + self.assertEqual(0, d.events) + self.assertNotIn('down', d) + self.assertNotIn('up', d) + self.assertEqual([], list(d)) + self.assertEqual([('down', 'jump', 'up')], list(c)) + + def test_copy_initialized(self): + j = self.jumper.copy() + self.assertIsNone(j.current_state) + + for i, transition in enumerate(self.jumper.run_iter('jump')): + if i == 4: + break + + self.assertIsNone(j.current_state) + self.assertIsNotNone(self.jumper.current_state) + def test_iter(self): transitions = list(self.jumper) self.assertEqual(2, len(transitions)) diff --git a/taskflow/types/fsm.py b/taskflow/types/fsm.py index cbe85b78..b8b6a69b 100644 --- a/taskflow/types/fsm.py +++ b/taskflow/types/fsm.py @@ -187,6 +187,20 @@ class FSM(object): for transition in self.run_iter(event, initialize=initialize): pass + def copy(self): + """Copies the current state machine. + + NOTE(harlowja): the copy will be left in an *uninitialized* state. + """ + c = FSM(self.start_state) + for state, data in six.iteritems(self._states): + copied_data = data.copy() + copied_data['reactions'] = copied_data['reactions'].copy() + c._states[state] = copied_data + for state, data in six.iteritems(self._transitions): + c._transitions[state] = data.copy() + return c + def run_iter(self, event, initialize=True): """Returns a iterator/generator that will run the state machine. From b84b76c557669056976c052d6cad06468e3fced7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 12 Sep 2014 00:05:34 -0700 Subject: [PATCH 021/240] Work toward Python 3.4 support and testing Change-Id: Icd0a426e3dd8a44c64e8ba337b36131392715c08 --- tox.ini | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tox.ini b/tox.ini index efc7bc3b..09488ac9 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ envlist = cover, py27, py27-sa8-mysql, py33, + py34, pylint, [testenv] @@ -72,6 +73,11 @@ deps = {[testenv]deps} -r{toxinidir}/requirements-py3.txt SQLAlchemy>=0.7.8,<=0.9.99 +[testenv:py34] +deps = {[testenv]deps} + -r{toxinidir}/requirements-py3.txt + SQLAlchemy>=0.7.8,<=0.9.99 + [testenv:py26-sa7-mysql] deps = {[testenv]deps} -r{toxinidir}/requirements-py2.txt From 2dec287d4e08a14b435ece8777088c3a59489db3 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 28 Aug 2014 14:35:02 -0700 Subject: [PATCH 022/240] Remove the dependency on prettytable Instead of pulling in yet another dependency for a mostly debugging feature lets just have a tiny table object/class that can work fine for our usage instead. Part of blueprint top-level-types Fixes bug 1368975 Change-Id: I21b1bd8152e5bf64b9060d3aabf4384350d05c0f --- requirements-py2.txt | 2 - requirements-py3.txt | 2 - taskflow/engines/action_engine/runner.py | 32 +++--- taskflow/tests/unit/test_types.py | 21 ++++ taskflow/types/fsm.py | 8 +- taskflow/types/table.py | 128 +++++++++++++++++++++++ 6 files changed, 169 insertions(+), 24 deletions(-) create mode 100644 taskflow/types/table.py diff --git a/requirements-py2.txt b/requirements-py2.txt index 9b204ea6..bfb837e4 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -18,5 +18,3 @@ stevedore>=0.14 futures>=2.1.6 # Used for structured input validation jsonschema>=2.0.0,<3.0.0 -# For pretty printing state-machine tables -PrettyTable>=0.7,<0.8 diff --git a/requirements-py3.txt b/requirements-py3.txt index 63880b31..1e1052e5 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -14,5 +14,3 @@ Babel>=1.3 stevedore>=0.14 # Used for structured input validation jsonschema>=2.0.0,<3.0.0 -# For pretty printing state-machine tables -PrettyTable>=0.7,<0.8 diff --git a/taskflow/engines/action_engine/runner.py b/taskflow/engines/action_engine/runner.py index 7a0b9c87..d2a17697 100644 --- a/taskflow/engines/action_engine/runner.py +++ b/taskflow/engines/action_engine/runner.py @@ -47,23 +47,23 @@ class _MachineBuilder(object): NOTE(harlowja): the machine states that this build will for are:: +--------------+-----------+------------+----------+---------+ - | Start | Event | End | On Enter | On Exit | + Start | Event | End | On Enter | On Exit +--------------+-----------+------------+----------+---------+ - | ANALYZING | finished | GAME_OVER | on_enter | on_exit | - | ANALYZING | schedule | SCHEDULING | on_enter | on_exit | - | ANALYZING | wait | WAITING | on_enter | on_exit | - | FAILURE[$] | | | | | - | GAME_OVER | failed | FAILURE | on_enter | on_exit | - | GAME_OVER | reverted | REVERTED | on_enter | on_exit | - | GAME_OVER | success | SUCCESS | on_enter | on_exit | - | GAME_OVER | suspended | SUSPENDED | on_enter | on_exit | - | RESUMING | schedule | SCHEDULING | on_enter | on_exit | - | REVERTED[$] | | | | | - | SCHEDULING | wait | WAITING | on_enter | on_exit | - | SUCCESS[$] | | | | | - | SUSPENDED[$] | | | | | - | UNDEFINED[^] | start | RESUMING | on_enter | on_exit | - | WAITING | analyze | ANALYZING | on_enter | on_exit | + ANALYZING | finished | GAME_OVER | | + ANALYZING | schedule | SCHEDULING | | + ANALYZING | wait | WAITING | | + FAILURE[$] | | | | + GAME_OVER | failed | FAILURE | | + GAME_OVER | reverted | REVERTED | | + GAME_OVER | success | SUCCESS | | + GAME_OVER | suspended | SUSPENDED | | + RESUMING | schedule | SCHEDULING | | + REVERTED[$] | | | | + SCHEDULING | wait | WAITING | | + SUCCESS[$] | | | | + SUSPENDED[$] | | | | + UNDEFINED[^] | start | RESUMING | | + WAITING | analyze | ANALYZING | | +--------------+-----------+------------+----------+---------+ Between any of these yielded states (minus ``GAME_OVER`` and ``UNDEFINED``) diff --git a/taskflow/tests/unit/test_types.py b/taskflow/tests/unit/test_types.py index 141cdfc8..0c13ee0f 100644 --- a/taskflow/tests/unit/test_types.py +++ b/taskflow/tests/unit/test_types.py @@ -23,6 +23,7 @@ from taskflow import exceptions as excp from taskflow import test from taskflow.types import fsm from taskflow.types import graph +from taskflow.types import table from taskflow.types import timing as tt from taskflow.types import tree @@ -161,6 +162,26 @@ class StopWatchTest(test.TestCase): self.assertGreater(0.01, watch.elapsed()) +class TableTest(test.TestCase): + def test_create_valid_no_rows(self): + tbl = table.PleasantTable(['Name', 'City', 'State', 'Country']) + self.assertGreater(0, len(tbl.pformat())) + + def test_create_valid_rows(self): + tbl = table.PleasantTable(['Name', 'City', 'State', 'Country']) + before_rows = tbl.pformat() + tbl.add_row(["Josh", "San Jose", "CA", "USA"]) + after_rows = tbl.pformat() + self.assertGreater(len(before_rows), len(after_rows)) + + def test_create_invalid_columns(self): + self.assertRaises(ValueError, table.PleasantTable, []) + + def test_create_invalid_rows(self): + tbl = table.PleasantTable(['Name', 'City', 'State', 'Country']) + self.assertRaises(ValueError, tbl.add_row, ['a', 'b']) + + class FSMTest(test.TestCase): def setUp(self): super(FSMTest, self).setUp() diff --git a/taskflow/types/fsm.py b/taskflow/types/fsm.py index cbe85b78..8df6461b 100644 --- a/taskflow/types/fsm.py +++ b/taskflow/types/fsm.py @@ -19,10 +19,10 @@ try: except ImportError: from ordereddict import OrderedDict # noqa -import prettytable import six from taskflow import exceptions as excp +from taskflow.types import table class _Jump(object): @@ -252,8 +252,8 @@ class FSM(object): if sort: return sorted(six.iterkeys(data)) return list(six.iterkeys(data)) - tbl = prettytable.PrettyTable( - ["Start", "Event", "End", "On Enter", "On Exit"]) + tbl = table.PleasantTable(["Start", "Event", "End", + "On Enter", "On Exit"]) for state in orderedkeys(self._states): prefix_markings = [] if self.current_state == state: @@ -287,4 +287,4 @@ class FSM(object): tbl.add_row(row) else: tbl.add_row([pretty_state, "", "", "", ""]) - return tbl.get_string(print_empty=True) + return tbl.pformat() diff --git a/taskflow/types/table.py b/taskflow/types/table.py new file mode 100644 index 00000000..d8d7f3b0 --- /dev/null +++ b/taskflow/types/table.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import itertools + +import six + + +class PleasantTable(object): + """A tiny pretty printing table (like prettytable/tabulate but smaller). + + Creates simply formatted tables (with no special sauce):: + + >>> from taskflow.types import table + >>> tbl = table.PleasantTable(['Name', 'City', 'State', 'Country']) + >>> tbl.add_row(["Josh", "San Jose", "CA", "USA"]) + >>> print(tbl.pformat()) + +------+----------+-------+---------+ + Name | City | State | Country + +------+----------+-------+---------+ + Josh | San Jose | CA | USA + +------+----------+-------+---------+ + """ + COLUMN_STARTING_CHAR = ' ' + COLUMN_ENDING_CHAR = '' + COLUMN_SEPARATOR_CHAR = '|' + HEADER_FOOTER_JOINING_CHAR = '+' + HEADER_FOOTER_CHAR = '-' + + @staticmethod + def _center_text(text, max_len, fill=' '): + return '{0:{fill}{align}{size}}'.format(text, fill=fill, + align="^", size=max_len) + + @classmethod + def _size_selector(cls, possible_sizes): + # The number two is used so that the edges of a column have spaces + # around them (instead of being right next to a column separator). + try: + return max(x + 2 for x in possible_sizes) + except ValueError: + return 0 + + def __init__(self, columns): + if len(columns) == 0: + raise ValueError("Column count must be greater than zero") + self._columns = [column.strip() for column in columns] + self._rows = [] + + def add_row(self, row): + if len(row) != len(self._columns): + raise ValueError("Row must have %s columns instead of" + " %s columns" % (len(self._columns), len(row))) + self._rows.append([six.text_type(column) for column in row]) + + def pformat(self): + # Figure out the maximum column sizes... + column_count = len(self._columns) + column_sizes = [0] * column_count + headers = [] + for i, column in enumerate(self._columns): + possible_sizes_iter = itertools.chain( + [len(column)], (len(row[i]) for row in self._rows)) + column_sizes[i] = self._size_selector(possible_sizes_iter) + headers.append(self._center_text(column, column_sizes[i])) + # Build the header and footer prefix/postfix. + header_footer_buf = six.StringIO() + header_footer_buf.write(self.HEADER_FOOTER_JOINING_CHAR) + for i, header in enumerate(headers): + header_footer_buf.write(self.HEADER_FOOTER_CHAR * len(header)) + if i + 1 != column_count: + header_footer_buf.write(self.HEADER_FOOTER_JOINING_CHAR) + header_footer_buf.write(self.HEADER_FOOTER_JOINING_CHAR) + # Build the main header. + content_buf = six.StringIO() + content_buf.write(header_footer_buf.getvalue()) + content_buf.write("\n") + content_buf.write(self.COLUMN_STARTING_CHAR) + for i, header in enumerate(headers): + if i + 1 == column_count: + if self.COLUMN_ENDING_CHAR: + content_buf.write(headers[i]) + content_buf.write(self.COLUMN_ENDING_CHAR) + else: + content_buf.write(headers[i].rstrip()) + else: + content_buf.write(headers[i]) + content_buf.write(self.COLUMN_SEPARATOR_CHAR) + content_buf.write("\n") + content_buf.write(header_footer_buf.getvalue()) + # Build the main content. + row_count = len(self._rows) + if row_count: + content_buf.write("\n") + for i, row in enumerate(self._rows): + pieces = [] + for j, column in enumerate(row): + pieces.append(self._center_text(column, column_sizes[j])) + if j + 1 != column_count: + pieces.append(self.COLUMN_SEPARATOR_CHAR) + blob = ''.join(pieces) + if self.COLUMN_ENDING_CHAR: + content_buf.write(self.COLUMN_STARTING_CHAR) + content_buf.write(blob) + content_buf.write(self.COLUMN_ENDING_CHAR) + else: + blob = blob.rstrip() + if blob: + content_buf.write(self.COLUMN_STARTING_CHAR) + content_buf.write(blob) + if i + 1 != row_count: + content_buf.write("\n") + content_buf.write("\n") + content_buf.write(header_footer_buf.getvalue()) + return content_buf.getvalue() From 185199b910d0401ce4950517088813423572fa06 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 14 Jul 2014 21:32:58 -0700 Subject: [PATCH 023/240] Add existing types to generated documentation Also makes some docstring adjustments to make sure the documentation looks readable (especially the inline examples). Part of blueprint top-level-types Change-Id: Ic6f02ce92449ee23aa9be8645edd8f6f11ee18ab --- doc/source/index.rst | 1 + doc/source/types.rst | 33 +++++++++++++++++++++++++++++++++ taskflow/types/fsm.py | 21 +++++++++++++++++++++ taskflow/types/table.py | 18 +++++++++--------- taskflow/types/tree.py | 29 +++++++++++++++-------------- 5 files changed, 79 insertions(+), 23 deletions(-) create mode 100644 doc/source/types.rst diff --git a/doc/source/index.rst b/doc/source/index.rst index 3e9326b6..23ffcfcb 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -91,6 +91,7 @@ Miscellaneous exceptions states + types Indices and tables ================== diff --git a/doc/source/types.rst b/doc/source/types.rst new file mode 100644 index 00000000..a5afa675 --- /dev/null +++ b/doc/source/types.rst @@ -0,0 +1,33 @@ +----- +Types +----- + +Cache +===== + +.. automodule:: taskflow.types.cache + +FSM +=== + +.. automodule:: taskflow.types.fsm + +Graph +===== + +.. automodule:: taskflow.types.graph + +Table +===== + +.. automodule:: taskflow.types.table + +Timing +====== + +.. automodule:: taskflow.types.timing + +Tree +==== + +.. automodule:: taskflow.types.tree diff --git a/taskflow/types/fsm.py b/taskflow/types/fsm.py index 8df6461b..092e26fa 100644 --- a/taskflow/types/fsm.py +++ b/taskflow/types/fsm.py @@ -247,6 +247,27 @@ class FSM(object): NOTE(harlowja): the sort parameter can be provided to sort the states and transitions by sort order; with it being provided as false the rows will be iterated in addition order instead. + + **Example**:: + + >>> from taskflow.types import fsm + >>> f = fsm.FSM("sits") + >>> f.add_state("sits") + >>> f.add_state("barks") + >>> f.add_state("wags tail") + >>> f.add_transition("sits", "barks", "squirrel!") + >>> f.add_transition("barks", "wags tail", "gets petted") + >>> f.add_transition("wags tail", "sits", "gets petted") + >>> f.add_transition("wags tail", "barks", "squirrel!") + >>> print(f.pformat()) + +-----------+-------------+-----------+----------+---------+ + Start | Event | End | On Enter | On Exit + +-----------+-------------+-----------+----------+---------+ + barks | gets petted | wags tail | | + sits[^] | squirrel! | barks | | + wags tail | gets petted | sits | | + wags tail | squirrel! | barks | | + +-----------+-------------+-----------+----------+---------+ """ def orderedkeys(data): if sort: diff --git a/taskflow/types/table.py b/taskflow/types/table.py index d8d7f3b0..b6257200 100644 --- a/taskflow/types/table.py +++ b/taskflow/types/table.py @@ -24,15 +24,15 @@ class PleasantTable(object): Creates simply formatted tables (with no special sauce):: - >>> from taskflow.types import table - >>> tbl = table.PleasantTable(['Name', 'City', 'State', 'Country']) - >>> tbl.add_row(["Josh", "San Jose", "CA", "USA"]) - >>> print(tbl.pformat()) - +------+----------+-------+---------+ - Name | City | State | Country - +------+----------+-------+---------+ - Josh | San Jose | CA | USA - +------+----------+-------+---------+ + >>> from taskflow.types import table + >>> tbl = table.PleasantTable(['Name', 'City', 'State', 'Country']) + >>> tbl.add_row(["Josh", "San Jose", "CA", "USA"]) + >>> print(tbl.pformat()) + +------+----------+-------+---------+ + Name | City | State | Country + +------+----------+-------+---------+ + Josh | San Jose | CA | USA + +------+----------+-------+---------+ """ COLUMN_STARTING_CHAR = ' ' COLUMN_ENDING_CHAR = '' diff --git a/taskflow/types/tree.py b/taskflow/types/tree.py index 41369b04..b6527422 100644 --- a/taskflow/types/tree.py +++ b/taskflow/types/tree.py @@ -107,21 +107,22 @@ class Node(object): def pformat(self): """Recursively formats a node into a nice string representation. - Example Input: - yahoo = tt.Node("CEO") - yahoo.add(tt.Node("Infra")) - yahoo[0].add(tt.Node("Boss")) - yahoo[0][0].add(tt.Node("Me")) - yahoo.add(tt.Node("Mobile")) - yahoo.add(tt.Node("Mail")) + **Example**:: - Example Output: - CEO - |__Infra - | |__Boss - | |__Me - |__Mobile - |__Mail + >>> from taskflow.types import tree + >>> yahoo = tree.Node("CEO") + >>> yahoo.add(tree.Node("Infra")) + >>> yahoo[0].add(tree.Node("Boss")) + >>> yahoo[0][0].add(tree.Node("Me")) + >>> yahoo.add(tree.Node("Mobile")) + >>> yahoo.add(tree.Node("Mail")) + >>> print(yahoo.pformat()) + CEO + |__Infra + | |__Boss + | |__Me + |__Mobile + |__Mail """ def _inner_pformat(node, level): if level == 0: From adf417cda526f4551079a0ae3fdcdeb8b803be09 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 10 Jul 2014 17:23:42 -0700 Subject: [PATCH 024/240] Add a mandelbrot parallel calculation WBE example The mandelbrot calculation is neat example to provide to show how the WBE engine can be used to compute various calculations (the mandelbrot set in this example). This will also create an image from the output if a filename is provided and the pillow library is installed (that library does the actual image writing). See: http://en.wikipedia.org/wiki/Mandelbrot_set Part of blueprint more-examples Change-Id: I12b9b7a2ce61b17ddaa2930cc9cf266ae0c60932 --- doc/source/examples.rst | 23 ++ doc/source/img/mandelbrot.png | Bin 0 -> 21971 bytes taskflow/examples/wbe_mandelbrot.out.txt | 6 + taskflow/examples/wbe_mandelbrot.py | 254 +++++++++++++++++++++++ 4 files changed, 283 insertions(+) create mode 100644 doc/source/img/mandelbrot.png create mode 100644 taskflow/examples/wbe_mandelbrot.out.txt create mode 100644 taskflow/examples/wbe_mandelbrot.py diff --git a/doc/source/examples.rst b/doc/source/examples.rst index 9199bc11..a6f0dd9f 100644 --- a/doc/source/examples.rst +++ b/doc/source/examples.rst @@ -163,3 +163,26 @@ Distributed execution (simple) :language: python :linenos: :lines: 16- + +Distributed mandelbrot (complex) +================================ + +.. note:: + + Full source located at :example:`wbe_mandelbrot` + +Output +------ + +.. image:: img/mandelbrot.png + :height: 128px + :align: right + :alt: Generated mandelbrot fractal + +Code +---- + +.. literalinclude:: ../../taskflow/examples/wbe_mandelbrot.py + :language: python + :linenos: + :lines: 16- diff --git a/doc/source/img/mandelbrot.png b/doc/source/img/mandelbrot.png new file mode 100644 index 0000000000000000000000000000000000000000..6dc26ee5e617d423977ab25d9fe70e68b8aee5d2 GIT binary patch literal 21971 zcmWifWmwcr7smfvbS&MlG)M|a$5PUbigXD`3rNS(-KaDRQWhaCAWL^6AWFB=APxII z@0Ype!+e-?UFV)N^E>y%>+7l!6VMX?06?svu3`uP;Qyvz0PnvMn50PL0)QvE8Y+s$ z{`vdu0@>@5b>hJaK>`m9;`@A(l4X39xP2mTCMtG!aSP?^-xc;f&nDuI z*$wIzOCwWTP`cbwy|~Nqmh!}*f?d(a~SZMx~GOQTkM=At`sjM0OmFy^cb|b+k7)J8o z4h6QB7VgQD54n*Ec&a!wk@@tY-KA3*v&0#%DC4v`>ZTZL0!Cjg{2fPDhE+`2utS!^ zxP+UQUlDT!e99EY*iDDwajdbJMZlHdjeN-Oy!K$E&yrc!s6&FH5ptYFR`M=ru?LFL z^w>UAM5((ikui>bc!0f$Kd>z0%7$^J0i?QE2g+H1juvm-zYGt3ICazb$U)2XbuuKd z!IPa%SQ~#YQmq1e)p|nCsC-cL8I*LW_HZ3KJk38h)!1JZ5(jN&nL6)+E`E7BII?$@ zpq_E^6sX%;BS&kvpH&4jCkcWMVzdPJ)KA@#|1g4pSW{=RfpuFgBAMMI+&&r*gagx_ zCpMH_!v0U1E@^rw$kPAXDYVn84&PdvO!y=+nIFVlc;9`P__C?5LcF5qJa5KmE0XLP zn0*)}gV9XMcaF4u~ zU6LHg$ATkmr{XeIy174S&PpFYKbws2M`qIz2d}e$=lR7iTUnguciXLPEN*_{Qoe9J zID+l6@Tln5AI3JKY%}Ot10>(==y*3w@eaDcLkRt4*_6^U!ZJ*M^rIZbx1(SV!0{9= zI#8FqeDhBh+leyR`fO1_!HqVwpG)!a!PY3r1Z0+lTlGtAmuq#8Hf~71%f97~f}(;Z z7&WB;WnE^(g^3OydJqrI!WFNIN|>7e6DB5%MWq8@dr`WMVvF_xqe_VJ;E%w*n=kjrAhqv@x_5X(z)|cf9Le zi?Zs{&zYZQ+@f!auNHx6cg3^-2-M0K4a!i*l)4N}*!`Lib~!%zNIUi5CHYP_K(m)I zmi#I3RVWMzboSXa)^pq9^!84@;t`i=aw>IgkyQ2T|MF*ncSU|DjB0E`;o1EX+Z`pf zO4YMAqy8-jcPSwNpr&#EXd+n#7tI>2r0|22d96)hm{Aai`~czCP8~oXp)A|Y{Bv4i ztMm7~@aag<0al937)Wqflq3=h9~eD78x=B*Pcmc2<0!W!4_?gsd#67qclVlng>a&y z0t}EtV_q%Cao|(@QM&4#dZW_-9535Q2Z%Am^#}Pq`*8*d7MWd7I9f`NjF;omv=DWjQ+SPy`_j0|No!OkUufu;KyL0GsMJ1lXT6d<72&?~+T- zW1BzH$r2aQKA&=beMiL?e!LDDN#f-~olvhlxIFJ165N*vM{wZSu)ukjpN3%W$yq6Y zuN@Ja=e*L(_h;o@-jc(>zR;y06azi0b_G6cOMI&N8Tpb*4UR(a@UIgTU9#%Bcc4lM zN>b;Wms?dZPK=%ba__WISlk^_U;?W;j3;agr-cp@!7zL@nO#l_XW95?W25np`EgJnm71B6E%aTuh%jC78scvqOAn4UAC6&4)GW612YO&v73 zESny_ipiI|rh4PpM>Xwy+d>uw^>KuBE6o38THu6&P}i(#?3QbG)6-THp*Wt#*f*R8 zUnxoUMm)`*4%6W@J1V8N!FU#S#sU89_(&3(sUGLg@paQ0mg&6AUwVa%&tE2ewm7tNHg&#pzt*2tL+!4ZgGdp;w=vM&o@nZVGA*b zX*MC@c>X5%!fb!KJ5;b7P+CQAt7{h8mG@N>?}Og9Ojdr20)5X0oJzF3@7j}E0YqUA zgLWs)D@>YN`$~h42+}*mD)C;YyMt+4;t~K9v%fHCx#*!;I;d;l8L^JSp2`V^043GKr{H=6iZqL<|FAHtKQFe(1Savrunw}MUtvjvNsq$ zOQWk<>f>c6PNFs%)r|{hPG^?8OmB?7&qJi0hr1Ba_de|K1iC{kKJ)1%L*XAKnby;! zvB{Y(G%oYCFS8yvNfrB4Q|z?;r^~>JO${~es0!2GYpYM4#YCRoAkP5Yq4lOWME`L9 zJMek%CO2^Pqj5D;aZ5N<#>5vpwCt!tRq#D|0~4@~7Z+{djiy33-hW)!io25+k5^w# zbOO)x)QncvNX(V6T{ItTF@GQVJ{s)B=~Po2El9vL9Qx1Sk#d3hio~Oq`est@H$^I?_%0T;W8GmZ47l$*wE#cEqh;;XO z2Q+b=XE(>)t1>Qsc^G1L%7F@QpLqHg&VJ_8Ln{XRoy#rghJA2NffTOjcpdhuN~!+9 zsKaSJoX_ek?H7v%Zg$Rn?;#P+MFRdIJT@A}{zIPf+w=(Jl(+#`lPSpNG~^vt;7spc zSk!;cTNHjnkx&cy7PK?~vH0!J4AE?Q5Z*+6@yJ{gJJqB(+p!S6WMSQImuSxP>Hdq7 zHL9+&nf5=G%Dml)v^?`@^4CB1f}gP)W?e>54TPjX^z9;?2W|Kw@5@;FSXM;NW%RJR z3i(~CIIz2qZq;or`9P(>Nk`u{~qq*Q27Ui-Y{ zOaH^vTAaOS{p|a&Gu7d85-Pa{Bs}fA(*(hX$Jk)8A?DHQ>8VFawzlJ4;NS4H=-t8g;Eeu7v0Kc5v==WSaw}>qk_#sr+>Yg}CBEM&s~h^o5TqrO!!5 zCmo46H+#!q@U)7IK}k*b5lR5`pN~DfZY^7Jv-w`dL)-zrWYt^~h`^}c$%wMC`_e%& zO`p%VJ@a6tejER*{}Bj566r)%v4#4eig{-U{`E5&*vY0+2iERdIoNoQpJh_<*k1{T zM76v@Xn{EZd7nWP&z7x^`gNj6cNMzH`kjawx812#qLsy*nkqX)e|wJZVdbWSs0WuC z9a?=b8h*xw+_-XylkL!3Km%E+&*Fty?<3R=CzWP48M5cES41=@1ZbeENN6h;tF(w_ znZo_&SiUb|2a(L1ho)en_hIYR(a#1wY}$^WX5i^_6g|UR&hy#t4uyRd(`XV5dmbQ^ zMei5~xG$5%xtfl6C&phs=eo-{Tz-Q0Pv^UXrs6b(Bo(Vk!q!B+uOgyyusYjv>=))hobqrn=>U9XhNh8e_Y{AqGlN~ugp>z9*2nXf2vYysV>_t zBOlnEfpQ!lKc_Ssq3XIgtEzOdJiQ-Z4(cbjhToZm3K6r8wnfvZsKjMO{XEPR(^J(F zNH+zewxxIlWidCbQ5~Rk7Iku_J2pJFtK=QJ@dP}5lTblI%1?j` zRW3gGxn2d1BS{O0)H2}A3InZHow&Ca+1oVkBCxoL<+^E;DYs4+YOv#ByB+=n&Hhx! z4!8%f*4fyvd3zL$nwAR+cU99zeh>9NOT9oC=5Yf2n*=CxFbKWi=&%`phbnUn|K~@e z8}ZvX!ru5U5&K*W$mPREZnr zVC=j7@mESgsBAz;94yYuQm#C68X*8IrZJVW*uHUNBG1YaY{n>spSpGa$xNH4y)+2j zcb-f0_C`V=zq=n}IL`Di69jK_036=L?Msg_k!Y75_5Hg%0Vc+Mp;gpIp(Hg%>-^L8 z`K_&m5p1ed1n|1Zz2l4amIX(Y5L_T}_pn7kre`PPgHD4F6#csl9 z#4{D2{aS2cyrVUN3g1P*76%|T5!eBcavtP~Zk=|J0>ba0qne?hzqOB$^1IVRMoO1# zO-jhmy1U{kMz14T+HPW`A-~OW_a?(nH$3EFjTh8PjD#^y)9GIuJH&TrZyv@B{W;@l zkpHIhxYqv{yxnkBY@lg^Yw^PAdLQME?@8z&xxDq_Jqz&VmfLSi=FR1ahAFUWDHE#* zRvDjWQP!s&dcsoAzIFjC)xOn3Mj=5Fah2Jr*9o&ShwU;~CN!9&=?*?E;E+pCPq;w5rDG9(e5xbEHhAIL=?;P~N} z3y;|FD0V4g-wU_%$);tRHGBs)p=~g(jG|dp$>;L}V?wE}G=&rlIeYe=v4&Hss@$W{ z95f}Fpsga%ScCtQ)O+Q^6nT7jECKES(u0lz`0R-)@LA)8d%FiFjVOKObp4a(K&P#r zNY!}!-E&SwM(Ghw^9K@1@;kFNz$6F?tS!7AUtDjyZ^y645aHt0Tpvr&$P}X>cp^H& zD$d2XN1lQ5y7QS(3-Xn%zu^Se=M_J2v`yg{`@iQ*Y^4MXS!b!Bz$HOrX+k*J3aFS2 zBN7~n3+?s7zawL~Suh{;w3&F-Z?TW#8}u)!D^)Ogxazz$4Jw^9sl=_W@imXra(bN` zQG2}7$Pw9b#0Di3s=bR#((z8=Y|@>6A7qPSwzLS*Cj+7x=%|U1p2n~Df84RDnG9vyBxYrC;-0o69z}?&ww|%yh zKql*hDCbl;Vy)j=H9P0CG0D;L+q>Y)e^&8ZNi(!kb)pGmY8h33FMC*QCjJBj(wSPV z6{w?T?~^$Qj1**Xj+m8xj9+Ccbn_+=lzs{X)xX?Opa(g=&`NVvt+XzuUeAE{ITS(>n)>}ckdpSw~yd*C>Vsj)F8||7T zL)XjTVB(@IT17B%qV%6c#m-&CMDc#)Su?TcV*Wbv%zWjgcug7PK}1;C4>vCWKR!m% zM>$+u8ULUIrL`=xi91XQbbNQR^%g(G-))hmoyz|xtpe7?mnGZ zBa(#C(^=zaOw_Ohsk&`Lk(yXBumnDr=c3FdNez8_!CNMN3U1!^C#>m55v9oQHpi#aA|ggSWR?^AHO2PbLW3rd%tJ&Rty zRH>dXu_C}QSa)3iGvEE~{}&S@g&qJz<9h;j^scPBZd%i0eohv#73E&6x~E8r;H_hG z>jtl0qv#vBU!iok0WjX$-R5qu_jX6*Tv=*E+wc`jVZ}OPRLaM_GU*hLm`HfH^@GvP z(AGEZlhPB;r~bZ;7+MgjJh?dtc-WyWbu(vmKBBoYu+|X-Myo*a2+*m5C%S8{zi5Lu zN#GB+!Pe9xGO#4U`&KFxhfh-$i-9lgM$lYHQ!n$Yf#2w3ou9wJ5N&yZl>b)z$64_8 z(t?dPA~PvT`Yi_)9Aj0Xr?pICmujpgAq3e;q>pK^jx<0*Fx}TEa|rw`K+k>lYo~_W zO=)yDU}d+Y`-gMB4leETc~~DT3w3yQMk>cyQH`hQG)c9Oje+ZsAnU9Jnb07e zK-J|W&;D1KpETOuSdKl)QbgJKFy{qmfD) z4{99m zCw^%49@C=pWUqdhzkLbjC@>VTW_!;~@sZBqu*9-#_sAAlb~RIQSreK2zm4d&D7FrCvRAkihzM#zvs^ zyroKxQQFgZmi*fP;p>Ay&fFR{ccJN-jLcjW6>WHZt~k|7He4@PvHEyfFcfwbcQ)UL z?h0zt1rxm)dJ-IDnpmNWzk2`NDc#}upJfSPiqw*_=dKUz1SKQ$X9b})F19rst6koP zzpEPd&ab(B`{fmVjNcYj-MPem8GWuM6cCT&h+`h2X2eHEfnte}?rm;uIi5`ZEzlha zTE1vaNP1kYTyEpnpN-6`^@(jv4O!0L7e5#PR_%*oqPA&7oT?+jkh_xZSjiEq`Gk=9h#QmaT1HGkJ?oJ0HEJc0IX!`cyL_` zWzJ5R=8CXtThvzqL}5MNG-b}Nds-eTL5%rv2-8(uvZ~*G&9j)LgR&3 zU)P{mMOo0NfyeRHT?3JB%$Y|o;1s?j!vl=&n-_C z#-c{P`yza7ers-&G?I43^!go*9T`5`HFIH_-XXND{VDgQV+H5NFq%ZIMX7Enqz}vp zW(C4%bm(NezJRZP`aGUf{bH$VpyIyFZgBde>SJxds~uCJd@k3&gqzrt{_g4J@SiHv zKbFGiB$IBpX{w2E3D@mZl{ptTPhoG*~=61Yf zRImK#)ODApqH}eh6)LFq<(2B z1OkHb;!$bevFl)YMR?YZCBUiEa}s_Y{va;tMWaYzR`QKHs`%{Cst_xcBN0|--@0(Q z!53jX&+!X7AglhfE6U$+Qi2VM$GW+;{@j-qoVLWQP(x&LY2D3Tm<0+{NA2#AnxGVb zPdRT1smQOA^x#P2nmdp-;e-;jzsKaZeg1(QinZU^ixT5$5W6!4&#!a&lN$+ z{NO@&u=Aog?jayoTdQ6qAls{L*hAIc&$Aj(yzK!(sqn9hT1M0pSRb1(gsYl_-)%W@ zPh&GAE^sGw>UjGVvO&YH+3fqgml?VQYHTNkFD5G?m+~vDm(1iqAaO~U@9&MBcS0~; zU?%bbu;9!H3azy+6px%};kE86*r6n{1^c@A`&`Gs5w%NS*C%I+_}C1AEEve=h)#QD zfg{c584$HdpTROn!xJk_JIk#Sc>Q{W4u@77o5G-CM8{>)3FV4;9WSenV$;=4`i6yd zssnK_QM^tKFM6r60T7IW?ZdywD^M0Dm-u24((E8p8AK(_b=Qg^QWU?10VY~(3J=fH z-zy*mH=3UFu<7pz6{&nRGCAVXRKSC`hqiv^y8smhm~JX4b3;ZD*mRtMFiS$mQ%jV3 z*9YLNATF-3?5seX8^XqB0}on$F$7ZI>*{sJCaX!4nMYDs2nLMQ2d4=m*+0ZicPv-g z{=+3iE&G{u7voSq^}^reWn(LgQ^%ZU7RZga(=9Rh|IsOD)UPr|?W}kDfw)KrL8aB7 zmBHF7*aze|+u7-^LC;}O7x+$Z^kizJtK*y!hhiThvjFA1r|&2iyw`56V_OcL+9whg zzf&zUR{h%I`kVkTM|yy5NDY+?SjY0x45fO3?`CWp+dLNG8>haW7l-)Ha_>Oz-+;EV zv3xbg2zzyK#OkmILVi zP1B88()y~ZI*T@M&Yde~uwH;5!7cfQz?Wxox9@~!3btZ)6E`fh1>&(P?rwZ5I26E# zuWNkq6i8!hkmkMyfb`o$nz^PssL|FH;KHzN#`0x3nTwC@0R z(`U;zd|LYy`SD^~9i{}=z!Px_G_Xy^^@SlRU)gStFuT!@WBPclAH&f4omEPXC)A4G z4KG{N4WUJq9Nj!Or_C|esOjz+qv5rj>Ha3Uu@Y8SNdvc?IZf1~*^&X!hl-W*_3yZt zt{kc8BVVx(4{PE<<#3=NtF%`WzxvzBzp)f+Iv(L8?HuOJ0^ZQwZ-*dj^|lsTo5^}5 zv?6Y{Cm3B`4aLlDMXY;taWQuhaS4urNo8+jgvm-lz@i#ioB8v60Y6XfBnvhUDgP;I zLJ%+&VND=!FScor@Ej2d7xDl~$+uj0`SdUFa)NYw0=pGGHW(dsy2P2_ScA{2$<_ zh^6c2c~j{doh~YXfo>7c51)bo{jZzV!xW zgv)Xb-SF|k%V39fbVY=G{ZmQU+k54jn)Jw}+A7dIO;))KA! zQVxPT=$kkb{g^2=+VJp&K zFQ|ckzf=O_@eqm#zsq%{OZ9a%Z5P=hGJ2Df$&f;OD)l!{Fkj;U;C|%qj(h1 zc8Ly)&MYK?-bZ&0=63p7as6l6Nl(MdKgNMB1e@|)`N)knj#ZljC!L0{*%}G|YwMb= zo+g1juxazE2-oWFKJe(eV!FPp2oz1ULhA&{uku^X+9M&+rnd;-s!7SOxD9TJtdPOW zxCu2J9OVS!I9N?I;>l)PEz_jq6auqMF2_e?Ej&?-MiogK%1Ci4%bxH_r3gUPZvG%7 zN-zRXm2{ts1fwj)kFme`)dI~{aW1byH~8NJm(4~D3MN5()~l(uM^i74F!AdjM^LGB zBr)J;8mRAi$VaWN&M^D?G)e@3uBruX>1T5h9uuP}evQqTBKrV}w)(T*?(f-Y0a}xX z?u-&;7H=!ntYEe>`{^jgFmvte!cg>vuKz|8Rflke0qbL>8n3$1^}ZOdxct{X7^Zgy z>4Fl_0y(^l=lawG9FR)te`NH#no8K{gduOpt-re7U-CGU&4a*bhoP4} z{ood;aptwb7${6fCBl%(TAF)KLGUdT)4ObX7p@T!RPmKj`euAf@e}YwF{|32;j=L#{Wdt z>6NCCb$gNZ_WiM5tjAAJ*>+L9G};Ip;_Q0Q^^rqO<3DvHT+0+}{JdYB7<5)!CYS2e zK#o}5QLvalRoW!hHU9Qf^l$$bvowp=`7vk;^BwI42VVGxf^bpuW}-b>`VlV#-v(FrF{Vt?+Uy+9UA4U9OPzIxa<5#lOp}SdX=TW^6U!D>Pvm>G~hGn`;(eu)=S{df#WCI~J{?l`|mRCq}x zNQS)h0i)+3VT7!qgedxr&yDFzxXNQ1SaAsQI=SK3#i==^A8DGB>}}&BrHRe(sVfu+ z`5Id>#_-&OzV(L+wyRJWw4!;WOPaCUL|2(IC#C$mo;!t-7f3VTP|dw{%e}0qanlAO zb$*yx2{L1e=H};F#k>#FZgYqyL=vK2(k|*~MGXlV-)_)_4s~X{W9kPcKcjE}0 z`^LqE^ZCj{SbhG%)#SISYTjrS%;ayW&B;+E-v_iQ*11!z>G&7R;umJ@m?Iu zKMsuk=g@Q+P%8dLt{y+~<76c3DkS~M(ba-vjTLjc$NUz_s%Wt|Oic9^A_B3?_dM|g zd4l^w^|Y)55fy!<*mFdkdiX`*;q9~UijA!bAjI&P06Fp4mAeDwzc`xP!>}Fb+S=hE4?UOe`A+RiJ|#E(R`Q(rbt8hnQh)=5sKgI(@nRbZ1?FJ#{@1Y zOP!d_!3I=|8T^J^B!a8ga`S(#0e@#Mq&UMq3U&g{ad2OJZ3lLA{amo?xa{yfGSCgczd>AjbW`2*|!eu^Sju-~+Z9 z!z=7K;=15V)__{dnmj#M6l8suruW_hG(n7g`$`8JjqqDgs)Fbn;KCCdNP$Cc-Ae<> zFJy!;6@Y^0*cO4`XT(AM>2hGB-@1if0&(^%Y8vFu4qAz+*LVgZ0?%{&QE0(l^gCjr zR~@&R0YZ~k-~w9er|PtXc*3HdP$R~#4TN?uwdy&Q7!Z+ zcZ7W@#JWcEA>B<*h44^M7CLGtKL;-diis*BD$fo#(Z3C1@+Zklz2stBvATSE`yheg z_R10!l?GLQHE31<|=CkmUE3m)oV@eNFbifV@#A~HaOa!Av5BU1m}2yw+=s5 z7KFX8r1Ofdcps02HYEWAnr5WOgG=sTZfN6{+>0OF893xWb)d)qxec8^(7ocks`X4C zVDhlTF>zA4K0A7N%N;|iQ8L)=YiP`gJQaS?s7F_;C4yPNDekNZ?SUya${uN89Z=r> zcSTpfKI?!uNqF?noxF#FhYIkt zoKac4<7?vH>d_-^Znrs~sh0Ax4h2P#6Z0uPEIT*$BlLI**>PmYr>pE2bHpjk$23u+ zE8aU`2;cv0z3FAOS~A`>|8aH+dDOKK^Bl!+{xK(~;IV!IE~ZU~uQusDj_CK^V-FHZ z2Dd1Q{iyr*FdqJ+Z4EAH+^`% zw|S{Fvfp$9yv=?+d*G-Urqz`7^0^>c(3T$x#ntBKFa!SjVx@(8Jm=(ZAo7ZV7vCh& zBo=@}`tA$Cxi!m#52SXr6N0HQI{PmifP5-2u}JA9sh2=fQ2QWm>mPcK<9HWsR!d#Fiz!Vw~s%?`waW4Zp_)vgI6^?J*)< z%d;!eO35vs!^28`v7*7x>h0*AMDRcuh9zR#ZRw$&tJ=)vax_j)F(uJRF_cJ$VOv_v z7)QNjQDk0NG&{n`R1@fUjl!Xo(c1D{S?ft>ng%37h{Qh#v34^?j9Bbl1cY-%t)zC! zDA5chbP`SMeO{WRz=4j4k4a?qjC}xzPAUeNZjbOB7#YGgw&8(n>!_#HmGtS*^pT5L zy4Vp5pLM--nag86#KzdFQ60!i7-qk*Zyi1*k#SEEZo_m|Ju{;|pMx)n@*s0_F?LlS zLXLiA5aW4ePy8jOvrAWa1^1*vaD;Bqf+!e}4LmU!CGDt}Yy;*){?KVFV!etPV;7AD zm&nb)ng4w9RSpySgIVJP{3$oU+Pm~b=O@~cn*EcsRo@^_&tGu1Ihi)Cb!pR6 zhlr7a9ooQu>)gAs=`8vyiT=4SSN?zTUZtGaSAK9}s0~-FO#RXtIzHJfimU{EyieJr4Pzk-iot|ZR^K=1h9UT6C$8>y6=x4XUEqN&pmB#!QFreqZbGyJ=!Xk`u+yHlNFkxt$&M(J zvUlWmoJipARV~>6yfF)+eWR$vMwS&}Y?0`B$d#)$uD4Mqqx+NP-PlGuX^c6LM;m53 zPNlUe&^O9aoC9J$|8qc|*8+AnmxZ0#=J=9W11jS#QFtJy9=i$c$e{DRgY+%OTxnC4 zRr?9beU2Mh^H730O1Z0RqaxcKD&3g=iC@O#)t7`&u8`1zexX|e{0SHI&wBH2o6Jm_ zzR&CPfHbq+HebRt(t(sEfvqa(ZmcPaE%Oefl3`m>&zjpbR)sN2Ay~{&v*V* zzgj|qa09)HxS!*AzLxPPH$!UuFvpj%#rH`jUJkZ*2ds85gI@7=MT)-1i0=!WpGn6} z4EVAirwmm+d>HSNr06PT=CDSia4ii7&_4v%UcQTXGm0k6{nqUFh~ikfSszwIJLw+K>|fl%*xifbZdVC(62Z#Ilju!g?^gZe1APhU0kbPQT_hd)95OdH=uGdPzVWm z!f_w7TmNi|Yv z+THa>=40aO^#^1tD_0R~1U(!P*V$ulpxRMdu(5&nO3U{Gof$F^_e%U^stvV=1$ZSZ;_$&Zwm~fwj z!GwqD7xUj&W$gP->Y5|8OgS;cTp$D4NvfJ?PL(a1Z0Z)_?VZfU)oR*9BKhODQZi|U zg^wX0{@G0{H>yk?%f+lW<3Yc zS|Nl7kc9nTvH}gMFEd)7ipDk&vipu zf2GL>s-*2$OAyfD?L#33S{Zp@O(KUN=DDh^(*^P|t}BLAr`)2VVYC!d=}t-x4t%S< zMO>!>V^i7NmkiS~)7p6b7lMgL^@%wLi66AX-tVDMg$`}Vwb;iHXuehA2E&V-wxC<8*RjQSoWd5y7HcV&WkGwSqQ5Gk~Z z@bG}90KMQF9(>7tA3%TzViJKA>~rnK$$Ih8-V_qUGA*Qz`}v%2FXg`)TO0-~GK$sT zDQ?D_i-Xv#dg;?wqx&dC`&r6Y1OP+=igwiz16&H6VNH#GrL$kW>*jF!4N=06=0+{S z;>wB1z2voj>(foVm)%a~K*(VtO8tZ%vv2%M@8tgI+(|Aj-YIVj^oUO|gDh_QZiI!; zngj4*H4Efi!>oBT3*DTqV3Gj@-dU2cDSvn-$h-7NZcS_?Q~fV{K4sAc>rb{aJD$Q8 zanQyWGf)i)ek!BrOrh9n2iDIt@i9vgW>*q*PjK~!pn?0GYbBc&XMU?+HN!fC0|8Kac@SkLZOri^{77sJ{Z zW8=2Ay8StD*pU~7Qow3zw?D0OXT&zhr!qvlTV5Kbkw4a^=O|hBJwn#a{P)o-EGmYs zPnga@7(3D8Rpg0;GZsb^+^y^!y0#A7Z_oJNemWABJG#WJ2@Yi)=*1QUBz@|0v}!uC znMQU@si0Dl98ikI>mrWF!$pJS0(PhImn=Bkvn=qmo~Uuc**uPh&^;#53ZgioY2W!9 zMHd|;|NYNP#+AMNJ#CriAB92#YOicl0L#6iq!;q9LbvMbl~m3dHvK^8cktAgI0Lmw zt6FtA82YrQ^H}SFN=D3vtZ$*6-d1~u#gE&F`YsTe5<^S| z)_F0E4-Ub}PW_gxpQKp*WZ|U0u2lwG_N=aM!z`P6wua?Pc(+eZO-unXBYE&PW0SvO zFGtEZks|Ns)plUlT)9uu|GXpr^bcA@?Htn~83$8S;+F3Zczl>Vh={4>ka+n<5ZwyD zm2Td!7W@ulC=`QRc(Qo^mHi>@XpEOub-PSp^~3`)c@u_Ey7q&;M7m-GDA$stE{ZkeW7!w<^;-WC#{-=& z)-SzN_K!;=_aq%pSd%Xbp4cc@r5w(lsup)S(>N0P;n&Pgm4_Y0H{*Pkqykz={1Du0gL!W6cA;d3?ys@9Feb%7!d_%rUh+ zPU<92aFlT&lXv{BVbs$h2P*;ODz@N5CSQ?2b_1#K>3@4l%1QmTZwXOzIDd*L8M%8M zY#u6J!$N3G@a7G`idF^#u{>=SO^0p|q&?wEPva zY@SbsvgL&u>qgix)#&&3k53oF{#B;(u?X0;UL?N=<>ACMjflsQJQS__*_UT6u}bM{ zkya`H4f)RbI59bSjQ%`ZH&jg0986)I&Bm1|SwkvDC=5Yo23eoRS2%U`B@lVW<9d=( z^E@$Qh4%k>R^jCL&*f)5cJ4<8p$f4uS!P+gUjSuSj!i1jQJx07h3XV(|63sX!apP! zbxNT)hqtVPK2RbOR@=3lta;1dYTvc;Fe76$;%GWTO5xZ!e7{yXx#hg2FOI_hDM9^z z&2%l+cQg^BKSVErr);$XI5bhry`qRy@7fO_*0-qlVdZBvzgP-(;*^H~w-i58w z@`Q3k&w+{hBH~(n`dHVQhWRl_dsUY11jDF)AmXkMxG)P=@zJN+T1@=JXkfKq|-?2lnBmObmj z;1GaJF@RJl%$gSx)1=O+I#YJUTL^Qs592kvWnjEALZ2k z+WzTIU@rWIK-gv|oBFk|R0nCo_^^c8-PHH8xs3Cj8!wsX-^%bY=fsu+-f{PoMqEiw z%fKappbOI)3JEz7Lx8I-S9}88L&+`c$Cy;Lu@R%q$614Fi1b^T*aq#D(ccszM%Qm zt&$LnKUSR5Pzxm4gj7`Yk(vjVkG(jHtJFYgI7HyEruCh*)ekkkM7Sm3!#l~%PHduq zJRry@PT${AGP~>Dw`a=if~zrq>=&do2a*Q8K1_Bim>YUNh`XDUy%;lyOw^iy-_doA9U7VCB&q0>xq-< zO=qQvh>1q14=QOzcLYh}<9er-c~mu!&Dw96+KWw-!+gvc+K){MGra(atKIO{6K^P3 z6m#oSGzJ38Z=5=qAgUhH7fSef7C_1kpkJS(=w|6p_Z+vu@~sE0{|lKFX6h|g@B*R+ z<#x*6-aL8qgdE_GgfJ|JQ0o`B`Rr~uHQ#n~_~f#u&4&gCm~<)_eHu8EU)=pUV2=;y zXXznCmW=A|z(ns;(!c()XRriT`%rzrQKC&Ww!-B8voG%^GGKr>jEMZ*JX@`U5yuK4 ziS~T}ZuxvUu00%j0qV_M)o_bNDBBGHF}-k-ah!<(AAF~d!k&kH5WdZs^zvWDFI4$E z85(@|uc)n=1E}y1<>d%OBtX)?QzpRSSE#vgFrYZgT`2(?ZkbsLOfar4*sKO{;h!Yb za1xP_*#aD!yIcwYOFM0rpfYa50P}G``l(tyU+s=O5iCL>`;dR6-ZwJ93~y%Ky@B?z zoZE!5atyGgd(9?sLh$@H(d5+hhHoSRI<|->X`(Izr1J04TQH6h?{_mLjs_~rz?qf- zt&Ls2AZ!l-#M$mJQ(jIi?)1d4;Vqds02*_IOdy(@CBQ7Rv_~gH;EOKSOCD`4u(H~fLrz?H?KrQiy)X~4Bi0)v~s{0>|d|r_16g1j(}ZDY?BD@Ao}5V z#!nniGPE+loI53LZ4G9DJ|cjh2Ua=-;t0lC({xBlZ$Th|qgNFHSCORc=BXlt3`1+P zV{t$c?_pE9d40s}s2;e>1nYzGdJ*#!_FEE%l0?70L=c;Z2A!8HP z{Z*zJk$1xYog8revokV3E2B+A0nO$xOF+rU0>F_0@cT!~05Re?)QAoM0?(T`K5PLU zA!6oM2)o)F@IKCqP6n$gsWrf6mVlC406^&C0qH$V2mmocV-?7VFob_Na#w7#;go5= z<4%-70B5X^{Y{hAv_$!h1mS|y$Xb*THGNQ40MI68E0A(D;~0rGfQYw@r4`=i2g%4^ zo`^>^1GeLSrU0Db>~RR>wpi-B!PYQ%dM^>yhQ;=PH+CV<8Em()^8v8oKX|v6nGV=I z`%+M8Ltr9y=wg6-^P26;0&SB3K{-bYQP=mFt+hJ<1+8oXcmTi0zjPK0mKlhFKw9R5 zeq(0Kp4lGE^ZJ`=aSa@E2Bo4;jU|1Y{xw*fj|$Q>%{`i8^izm6qq3P4CvsM z_u(wNEN$;t8eq~ma*McdB*h9~1z(%f;rWFD5JRpIgv^E^=SX>bS4tqzhtvCvg`yD8 z%InZ0>#ZeGrcJ~)wg7~5M+Xe=C&@5b=*|*A9t_YjVTY|Bf4-!?EAatGX4clm!HE7+ zZRnN&^Unc&+Pruk;62AH5GWhuCQ=@N044$h!~{D*44G3dRCbk1Pw-xVBvGsYm=S8Y zU{-}>I#bO6a|Vg(iyj>~zaD7T16T3M+NX6MJ=Ufscb)o;^AxLqJUC^XB{g=dKzn-K zGQV*l-~Q$`IHLpHLIm_`^fD>1cLp|>tS0ubH&_DDKgn=bZO&Y}DPYfF z#NIxyGbVu(GA?KA0U$~q+6pGv{UUNK-ej^KK0v!~xH}!7H_p6>`p9AEpT>N^!jk|Z zV_}&0fIaCb&qODXsa~Uml(X^Pm~09V^2slCo;kG$rN0R;sdgsVa_a1a_O(yqE#}(5>X%{?wu|B zYMfgJ-eeBw+v?DHfB{0J4!(jl$(aAJ!20iW%n`84bpSsk+u*p{7w*=W*ye@@4~Oh? zfx_8ykbJ3My=sf@%nc0;a2uLm)`zZgZC>I81qdSJ0)+q{iLxaCCRK3N-GY}+8eGB# zvGD-F^Fqd(lG5d`)W43Yx>0-%#{*2}ISk;RP%knO&Q1aM<|SAy*HN&h1`t3H!4d!# zNuG9Qj*N>QV=W3kI&rwwnxOz7ZdAw4z|-0M!K{t3F#w#jvE0;emqW>JLqsNL>#G+5 z=U(Jp>Hxq1#v90?E9v2YI2Mpj&0@~iU>0VFbiOW_gggxl*(C}sfY{N|Y4mK_Ss$Gd zswCg@EWrRyv0MHXm4GRoH13uG`lBQDR9XVbEGwiRudy?TqclVeXRC5|hk5(uQcQR& zFzf9B01jpkz#M=O(CwpA$1y_S5DQ@U$q@kXyQ@4oj~G=u&&YXz!9cX1BywU4nizmP zzRiaL?Y6Q&iBv>_XtTyPX0X{qNFSWgKdZ%pGc%zDbNmYcd_HL)HeqZJp=k{-rrDUi z-XBg3z*uVlAao`Mv9gGOX#G}L-W3y_B_D0TOXs}C=>tq=1qAL|0t|WXUL2r^Hi$!? z8NN1T`KaG@ZYyerf=vRH+Yimm`kn3Z#j({Z4h($6Q8ov(7z1n+t93Ee*`xyC?zV$V z7_=a5E?ig(((4m%bz}g7X8u^Lg5ae=&0Yrk74|DOZb-f}-kIG#kzf$0eM#uEn^PFZ zy?pOIu`$EE5d8&x!dq^1YIp)BFe5zadF)$lO26XMG7bBDGn!2(_b?~?AY2?`sY9XXxz2| z%HOpF7>ag@!2~FeG}r?6<;gH$1pWcpTEW1Y7chctky6e~V+Fe7i}_$voKpVD8PMW2 zT&Eknos^+}PTyubs(9xNFwiR5w0=q^0rodrji(d9q(3bHNt!&X3kz-n!2IeN0C;gS zShz}=z|%eiD-nXf`$zvEPj0^Rtytb(65U6M>Me*e|@YxOB_XdE8ebVGR; zC0u%1;Jg2t786fT9`;dKo;flSi)`#Bhja-9Qskd;7z+_FC^e@@>3Y}I_aS}Px+|_870QV#UhPPe6LBJXQs5BrK#3x%<9jW$F zB;z0Id!J3AcTUd!+6n=wK>wCziOAplSME4mDmIWCdKJH8L(s)&Szt5gW_PFWo2Y^Q zCG(v7lar#)rqC`eU0cLuzUPf;G~G|MGXGm?w9=b#(USpgw-UUh?b_?L>F=C-{($k; zatOCU0z2andN_>gH0@VWg-hGb){E|oKyz=>S&n)?rILWr19!40yWuAO4sLO zs{BnhkpKcoL4F^X;mChY_83rE^S6orV!sG26Tl=hKyL>4Y8+7bQhiZWzMOG@^oUly z+K#u05oH0bb$&hpiHb{L92GbwNRhE9OHKXs%Z47W{^uDM& zO&ow0y|3xcYQd;punE>i@AdhGd(+=GkN=4%(tirLt#Rn0)+=H>-5IQ!{#u(W{tKXI zK1YsD4U4?W-&Qnxvkyk^n;*swaM|l9-_>|ttXU+~WB?Juesi-@?F!}@6k-OBe?F6DsIeeJoWT83QmhloJyzM&&l3lt;H zb@#lH_;U$>$I$_BIRp0NfX;VLAkcuM-d<3!8F1}Gp*zh(f4;FchyiXrEzHaT*{=+_ z__ykRR9wTa#(`+8zg6&G`J#v1-o6aHBcFh)QF@lMSwREF)$Gc-h8b>%`x? z!YPN|uqohCMZje;pr8eOW+!`kz-Z{F=S6qrU0UCa0^p%^0F1@~=hBSUVBX;MjUA*d zAC%g6SJ^fE+s#N5po0P0?ISnDI2>>`(QNID(H;=@GX$|YZ4o|q)ZDq~yY?LqyZ^x? zz!kWJ0}A-q1Ez1jDXm}4f7&m1^3lv=Y$yXft_bLD5NP_vf}vq;pAF#{Mc=aq8_-~z z&!A@>_<|l!0$jm@{S<-523W2X4EIq4rJ~d!s1e=tpL>K2juw7%5ilAD^hzy}{Z(6R z`*{YdatY zup#m{eyT9^>FgNq^pZLrRsxL70W}t=m7zXUfQ=Ey&^QS7WBa3WZCIDiT^I?MF~A*& zfRQ<%msRNe3~*w2W)~rl3labP?_z7yZvb3W$vq~F=!t9k)^M+_4DhT70T3X*3NHg!u zFVnCbuwX619W$8M5rnfHohQq!2h!yXaO<7}m|zm94_5Vm@(_lQzLX#^#~RS6rH?Ri z%n`*5W_}Oxr23_t>wJJ21i&>ZW4SzB)-o{`L#G$ynp2r9g(}B^K0;sG8cpnCaAr8yXE<-TDVg+!_Buksaj1Ql| z0I~S?Vf-DPCCeR*p=)%2JLo>9=|whT-T0M;8OQD66n+CUgZ4YH(Yi=~TEX%$O$hYM zga@f`MgW}YdjBW8;vJlq03Ac@9*hCkq3sC@GJt_k3%tNcUW66^$1xpDKnfc1}gh;Iwv``=gkf6XIc zuUiIwfk~k4=CURTuw@XW`^Ie8%Ahy}oDQyR5{!)P{|4MS1C00XuBz$EoIziJ1KLoE z)y4qe1)*Ya5(}PxMMcU=7m>;WX$nZX?#=ansOz5oYZ!t?-WqNw!C?E$ahN51VSrD? zQbfWlR>&XsKXT9!B&vYUIrcvEz^i)EcC*MXyS+fqi;Z=^6M>YJ#q=?8w8HzZBqv<* zPwNaYqJ{8ER_Y@G#W62|JlaX^*X?Ph=7UJ05G8AI7=N#cvVumO#uKn zg)zR_YDF`=nyZ`W#Q-zeGp@rF98ieb&TyF|lXSxp!D?wE6S^YOa3q}NuPOlQD!_;W z;QDv}HC6$Wp+G%Gk+wd391x`|LBnkJn?Awl?`r}WM+dkQi}1)!qP43GN}^9QaTpPN zFakCn>@RDXR@>D_DgZjPz%hwCLS6kB;LN-I!W>ZE7$k6-dW;}(WczagQYUQ~{7P9Q zxhod{^GyI>YzmlcV=%2a8`x|aynpXxm&dxIA49xv7=UFU zG+?CpWEqvXK$+2Cs;-Rk)<;Sk_XRkh4RsOVbbV`HU7qL@Vp<^E2%4>bTiG8jKr-vIBJ73`|+ zUgTsEeHmc9-OGtFbw9ZLg||BcguNok_v_lc0${|wfP0w)F7HR@Qm}H7i3wIP7!hRD zdTw~Cppc{S`B&KtDqvzxta3su2&@YPX`{`($jCBAOdNSP4|s7$F4*^Gux)fGs1?7< z31D0$@bNjI1>Lb780!1T4W7e@4>;;7;BGkJQVbFS*_!W!Ba(#V7gjw~EP2s7L&8(@Eo%mDWy0xs^z z@cTTq-r@U;d@4nUy~sD{1_EFR1B_|MoE}qjq+Tb1u-L~Rr&vA7ZxGZx2i!0H$DaZo zodcS0YtQaT!P2lkguZ%Va&aLh#n|t6h#}}+Y<)d3L3gWF5`ki91}hy$0D&|kKNem( zU39kc(YiSV<68pnhfOZ)$@#nT669H|J`96oWELK$X9uUl=ygx93b>m|V9cI0jb#D? z&)aJT_|wF{Hd1bwgsQ3mBMN}=-|ZB+gCCp&dfh|-AP}#E0AOY|Cf=@m^(2FbqW?5@ zfCuM*0Wk1>ZWz{D%h>Dz^Qw`p1O`|EZnnglW;gJ$^?%Gcg>_)Y(v5_)SwFZy7V z&xMaiwMn>n7z6Zvo%h25(}4tDeZ=HKf|4a|=xEqu3!IM+RmSb??EgOnDaX>ABDDzs O0000= 4: + return i + return max_iters + + min_x, max_x, min_y, max_y, max_iters = mandelbrot_config + height, width = image_config['size'] + pixel_size_x = (max_x - min_x) / width + pixel_size_y = (max_y - min_y) / height + block = [] + for y in six.moves.xrange(chunk[0], chunk[1]): + row = [] + imag = min_y + y * pixel_size_y + for x in six.moves.xrange(0, width): + real = min_x + x * pixel_size_x + row.append(mandelbrot(real, imag, max_iters)) + block.append(row) + return block + + +def calculate(engine_conf): + # Subdivide the work into X pieces, then request each worker to calculate + # one of those chunks and then later we will write these chunks out to + # an image bitmap file. + + # And unordered flow is used here since the mandelbrot calculation is an + # example of a embarrassingly parallel computation that we can scatter + # across as many workers as possible. + flow = uf.Flow("mandelbrot") + + # These symbols will be automatically given to tasks as input to there + # execute method, in this case these are constants used in the mandelbrot + # calculation. + store = { + 'mandelbrot_config': [-2.0, 1.0, -1.0, 1.0, MAX_ITERATIONS], + 'image_config': { + 'size': IMAGE_SIZE, + } + } + + # We need the task names to be in the right order so that we can extract + # the final results in the right order (we don't care about the order when + # executing). + task_names = [] + + # Compose our workflow. + height, width = IMAGE_SIZE + chunk_size = int(math.ceil(height / float(CHUNK_COUNT))) + for i in six.moves.xrange(0, CHUNK_COUNT): + chunk_name = 'chunk_%s' % i + task_name = "calculation_%s" % i + # Break the calculation up into chunk size pieces. + rows = [i * chunk_size, i * chunk_size + chunk_size] + flow.add( + MandelCalculator(task_name, + # This ensures the storage symbol with name + # 'chunk_name' is sent into the tasks local + # symbol 'chunk'. This is how we give each + # calculator its own correct sequence of rows + # to work on. + rebind={'chunk': chunk_name})) + store[chunk_name] = rows + task_names.append(task_name) + + # Now execute it. + eng = engines.load(flow, store=store, engine_conf=engine_conf) + eng.run() + + # Gather all the results and order them for further processing. + gather = [] + for name in task_names: + gather.extend(eng.storage.get(name)) + points = [] + for y, row in enumerate(gather): + for x, color in enumerate(row): + points.append(((x, y), color)) + return points + + +def write_image(results, output_filename=None): + print("Gathered %s results that represents a mandelbrot" + " image (using %s chunks that are computed jointly" + " by %s workers)." % (len(results), CHUNK_COUNT, WORKERS)) + if not output_filename: + return + + # Pillow (the PIL fork) saves us from writing our own image writer... + try: + from PIL import Image + except ImportError as e: + # To currently get this (may change in the future), + # $ pip install Pillow + raise RuntimeError("Pillow is required to write image files: %s" % e) + + # Limit to 255, find the max and normalize to that... + color_max = 0 + for _point, color in results: + color_max = max(color, color_max) + + # Use gray scale since we don't really have other colors. + img = Image.new('L', IMAGE_SIZE, "black") + pixels = img.load() + for (x, y), color in results: + if color_max == 0: + color = 0 + else: + color = int((float(color) / color_max) * 255.0) + pixels[x, y] = color + img.save(output_filename) + + +def create_fractal(): + logging.basicConfig(level=logging.ERROR) + + # Setup our transport configuration and merge it into the worker and + # engine configuration so that both of those use it correctly. + shared_conf = dict(BASE_SHARED_CONF) + shared_conf.update({ + 'transport': 'memory', + 'transport_options': { + 'polling_interval': 0.1, + }, + }) + + if len(sys.argv) >= 2: + output_filename = sys.argv[1] + else: + output_filename = None + + worker_conf = dict(WORKER_CONF) + worker_conf.update(shared_conf) + engine_conf = dict(ENGINE_CONF) + engine_conf.update(shared_conf) + workers = [] + worker_topics = [] + + print('Calculating your mandelbrot fractal of size %sx%s.' % IMAGE_SIZE) + try: + # Create a set of workers to simulate actual remote workers. + print('Running %s workers.' % (WORKERS)) + for i in range(0, WORKERS): + worker_conf['topic'] = 'calculator_%s' % (i + 1) + worker_topics.append(worker_conf['topic']) + w = worker.Worker(**worker_conf) + runner = threading.Thread(target=w.run) + runner.daemon = True + runner.start() + w.wait() + workers.append((runner, w.stop)) + + # Now use those workers to do something. + engine_conf['topics'] = worker_topics + results = calculate(engine_conf) + print('Execution finished.') + finally: + # And cleanup. + print('Stopping workers.') + while workers: + r, stopper = workers.pop() + stopper() + r.join() + print("Writing image...") + write_image(results, output_filename=output_filename) + + +if __name__ == "__main__": + create_fractal() From f8fbb30412edc41a1df05c7938db9dc973357f5c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 29 Aug 2014 02:09:36 -0700 Subject: [PATCH 025/240] Mention issue with more than one thread and reduce workers Due to how it appears the filesystem transport in kombu is not thread-safe we will work around this by not having more than one worker active at the same time in this example. Oddly it appears the memory transport is unaffected (but from looking at the code it doesn't look safe either), this may just be due to how the python memory model works though. Part of blueprint more-examples Change-Id: Idaf04fb1a6a622af292511bbcf25329c9a5aab53 --- taskflow/examples/wbe_simple_linear.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/taskflow/examples/wbe_simple_linear.py b/taskflow/examples/wbe_simple_linear.py index e28579f8..bfec2d86 100644 --- a/taskflow/examples/wbe_simple_linear.py +++ b/taskflow/examples/wbe_simple_linear.py @@ -53,7 +53,12 @@ USE_FILESYSTEM = False BASE_SHARED_CONF = { 'exchange': 'taskflow', } -WORKERS = 2 + +# Until https://github.com/celery/kombu/issues/398 is resolved it is not +# recommended to run many worker threads in this example due to the types +# of errors mentioned in that issue. +MEMORY_WORKERS = 2 +FILE_WORKERS = 1 WORKER_CONF = { # These are the tasks the worker can execute, they *must* be importable, # typically this list is used to restrict what workers may execute to @@ -90,6 +95,7 @@ if __name__ == "__main__": tmp_path = None if USE_FILESYSTEM: + worker_count = FILE_WORKERS tmp_path = tempfile.mkdtemp(prefix='wbe-example-') shared_conf.update({ 'transport': 'filesystem', @@ -100,6 +106,7 @@ if __name__ == "__main__": }, }) else: + worker_count = MEMORY_WORKERS shared_conf.update({ 'transport': 'memory', 'transport_options': { @@ -115,8 +122,8 @@ if __name__ == "__main__": try: # Create a set of workers to simulate actual remote workers. - print('Running %s workers.' % (WORKERS)) - for i in range(0, WORKERS): + print('Running %s workers.' % (worker_count)) + for i in range(0, worker_count): worker_conf['topic'] = 'worker-%s' % (i + 1) worker_topics.append(worker_conf['topic']) w = worker.Worker(**worker_conf) From 452652431a32ef4bc6dbb549efe858b0c3e71466 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 10 Sep 2014 11:36:21 -0700 Subject: [PATCH 026/240] Example which shows how to move values from one task to another Part of blueprint more-examples Change-Id: I05151c3f827d6a8a0b6a7f0aeab17bab2c8ce440 --- doc/source/examples.rst | 12 ++++ taskflow/examples/simple_linear_pass.out.txt | 9 +++ taskflow/examples/simple_linear_pass.py | 68 ++++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 taskflow/examples/simple_linear_pass.out.txt create mode 100644 taskflow/examples/simple_linear_pass.py diff --git a/doc/source/examples.rst b/doc/source/examples.rst index 9199bc11..10850190 100644 --- a/doc/source/examples.rst +++ b/doc/source/examples.rst @@ -1,3 +1,15 @@ +Passing values from and to tasks +================================ + +.. note:: + + Full source located at :example:`simple_linear_pass`. + +.. literalinclude:: ../../taskflow/examples/simple_linear_pass.py + :language: python + :linenos: + :lines: 16- + Making phone calls ================== diff --git a/taskflow/examples/simple_linear_pass.out.txt b/taskflow/examples/simple_linear_pass.out.txt new file mode 100644 index 00000000..1e58a63c --- /dev/null +++ b/taskflow/examples/simple_linear_pass.out.txt @@ -0,0 +1,9 @@ +Constructing... +Loading... +Compiling... +Preparing... +Running... +Executing 'a' +Executing 'b' +Got input 'a' +Done... diff --git a/taskflow/examples/simple_linear_pass.py b/taskflow/examples/simple_linear_pass.py new file mode 100644 index 00000000..bda25216 --- /dev/null +++ b/taskflow/examples/simple_linear_pass.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import sys + +logging.basicConfig(level=logging.ERROR) + +self_dir = os.path.abspath(os.path.dirname(__file__)) +top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, + os.pardir)) +sys.path.insert(0, top_dir) +sys.path.insert(0, self_dir) + +from taskflow import engines +from taskflow.patterns import linear_flow +from taskflow import task + +# INTRO: This examples shows how a task (in a linear/serial workflow) can +# produce an output that can be then consumed/used by a downstream task. + + +class TaskA(task.Task): + default_provides = ['a'] + + def execute(self): + print("Executing '%s'" % (self.name)) + return 'a' + + +class TaskB(task.Task): + def execute(self, a): + print("Executing '%s'" % (self.name)) + print("Got input '%s'" % (a)) + + +print("Constructing...") +wf = linear_flow.Flow("pass-from-to") +wf.add(TaskA('a'), TaskB('b')) + +print("Loading...") +e = engines.load(wf) + +print("Compiling...") +e.compile() + +print("Preparing...") +e.prepare() + +print("Running...") +e.run() + +print("Done...") From c5aa2f94d15b89da119ba8a2b9a49a570e6d7357 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 16 Sep 2014 17:42:59 -0700 Subject: [PATCH 027/240] Remove useless __exit__ return By default we will return none, and that qualifies as falsey, so we don't need to explicitly return false when the default will do just fine. Change-Id: I93a858177be138bf09abfc32897c96a991f69386 --- taskflow/types/timing.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/taskflow/types/timing.py b/taskflow/types/timing.py index cd822ae7..5d179c28 100644 --- a/taskflow/types/timing.py +++ b/taskflow/types/timing.py @@ -86,8 +86,6 @@ class StopWatch(object): self.stop() except RuntimeError: pass - # NOTE(harlowja): don't silence the exception. - return False def leftover(self): if self._duration is None: From 6bbf85b5a50437a65a8ce2acba9eb73c5003ff78 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 8 Jul 2014 14:51:33 -0700 Subject: [PATCH 028/240] Add a timing listener that also prints the results Instead of just recording them it can also be quite useful (especially for debugging) to print the start and stop timings as they occur. Also adds an example that shows how this can be used and an explanation of why it is useful to have this type of capability. Part of blueprint more-examples Change-Id: Id2dc3f8dc9ac94e511470e39f499f325b33537ee --- doc/source/examples.rst | 12 ++++++ doc/source/notifications.rst | 2 + taskflow/examples/timing_listener.py | 59 ++++++++++++++++++++++++++++ taskflow/listeners/timing.py | 51 ++++++++++++++++++++---- 4 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 taskflow/examples/timing_listener.py diff --git a/doc/source/examples.rst b/doc/source/examples.rst index 9199bc11..365794af 100644 --- a/doc/source/examples.rst +++ b/doc/source/examples.rst @@ -34,6 +34,18 @@ Building a car :linenos: :lines: 16- +Watching execution timing +========================= + +.. note:: + + Full source located at :example:`timing_listener`. + +.. literalinclude:: ../../taskflow/examples/timing_listener.py + :language: python + :linenos: + :lines: 16- + Linear equation solver (explicit dependencies) ============================================== diff --git a/doc/source/notifications.rst b/doc/source/notifications.rst index 3fe430de..249c3a7f 100644 --- a/doc/source/notifications.rst +++ b/doc/source/notifications.rst @@ -165,3 +165,5 @@ Timing listener --------------- .. autoclass:: taskflow.listeners.timing.TimingListener + +.. autoclass:: taskflow.listeners.timing.PrintingTimingListener diff --git a/taskflow/examples/timing_listener.py b/taskflow/examples/timing_listener.py new file mode 100644 index 00000000..ab53a9aa --- /dev/null +++ b/taskflow/examples/timing_listener.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import random +import sys +import time + +logging.basicConfig(level=logging.ERROR) + +top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, + os.pardir)) +sys.path.insert(0, top_dir) + +from taskflow import engines +from taskflow.listeners import timing +from taskflow.patterns import linear_flow as lf +from taskflow import task + +# INTRO: in this example we will attach a listener to an engine +# and have variable run time tasks run and show how the listener will print +# out how long those tasks took (when they started and when they finished). +# +# This shows how timing metrics can be gathered (or attached onto a engine) +# after a workflow has been constructed, making it easy to gather metrics +# dynamically for situations where this kind of information is applicable (or +# even adding this information on at a later point in the future when your +# application starts to slow down). + + +class VariableTask(task.Task): + def __init__(self, name): + super(VariableTask, self).__init__(name) + self._sleepy_time = random.random() + + def execute(self): + time.sleep(self._sleepy_time) + + +f = lf.Flow('root') +f.add(VariableTask('a'), VariableTask('b'), VariableTask('c')) +e = engines.load(f) +with timing.PrintingTimingListener(e): + e.run() diff --git a/taskflow/listeners/timing.py b/taskflow/listeners/timing.py index e21dd642..4a08256e 100644 --- a/taskflow/listeners/timing.py +++ b/taskflow/listeners/timing.py @@ -16,6 +16,7 @@ from __future__ import absolute_import +import itertools import logging from taskflow import exceptions as exc @@ -23,14 +24,21 @@ from taskflow.listeners import base from taskflow import states from taskflow.types import timing as tt -STARTING_STATES = (states.RUNNING, states.REVERTING) -FINISHED_STATES = base.FINISH_STATES + (states.REVERTED,) -WATCH_STATES = frozenset(FINISHED_STATES + STARTING_STATES + - (states.PENDING,)) +STARTING_STATES = frozenset((states.RUNNING, states.REVERTING)) +FINISHED_STATES = frozenset((base.FINISH_STATES + (states.REVERTED,))) +WATCH_STATES = frozenset(itertools.chain(FINISHED_STATES, STARTING_STATES, + [states.PENDING])) LOG = logging.getLogger(__name__) +# TODO(harlowja): get rid of this when we can just support python 3.x and use +# its print function directly instead of having to wrap it in a helper function +# due to how python 2.x print is a language built-in and not a function... +def _printer(message): + print(message) + + class TimingListener(base.ListenerBase): """Listener that captures task duration. @@ -46,11 +54,17 @@ class TimingListener(base.ListenerBase): def deregister(self): super(TimingListener, self).deregister() + # There should be none that still exist at deregistering time, so log a + # warning if there were any that somehow still got left behind... + leftover_timers = len(self._timers) + if leftover_timers: + LOG.warn("%s task(s) did not enter %s states", leftover_timers, + FINISHED_STATES) self._timers.clear() def _record_ending(self, timer, task_name): meta_update = { - 'duration': float(timer.elapsed()), + 'duration': timer.elapsed(), } try: # Don't let storage failures throw exceptions in a listener method. @@ -66,5 +80,28 @@ class TimingListener(base.ListenerBase): elif state in STARTING_STATES: self._timers[task_name] = tt.StopWatch().start() elif state in FINISHED_STATES: - if task_name in self._timers: - self._record_ending(self._timers[task_name], task_name) + timer = self._timers.pop(task_name, None) + if timer is not None: + timer.stop() + self._record_ending(timer, task_name) + + +class PrintingTimingListener(TimingListener): + """Listener that prints the start & stop timing as well as recording it.""" + + def __init__(self, engine, printer=None): + super(PrintingTimingListener, self).__init__(engine) + if printer is None: + self._printer = _printer + else: + self._printer = printer + + def _record_ending(self, timer, task_name): + super(PrintingTimingListener, self)._record_ending(timer, task_name) + self._printer("It took task '%s' %0.2f seconds to" + " finish." % (task_name, timer.elapsed())) + + def _task_receiver(self, state, details): + super(PrintingTimingListener, self)._task_receiver(state, details) + if state in STARTING_STATES: + self._printer("'%s' task started." % (details['task_name'])) From 97e6bb162cb91b4e0bbb14c9a40824ef35cd98c8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 18 Sep 2014 13:09:42 -0700 Subject: [PATCH 029/240] Link a few of the classes to implemented features/bugs in python This adds comments that associate the classes we have for threading usage to upstream bugs in python where similar features are being created (and potentially supported upstream). When we are able to reduce the number of supported python versions we can/should try to remove our implementations and move to the ones that may showup in the python standard library instead. Change-Id: I7b58380aeb57a58fa3b3c424c9f39de30f44f0e9 --- taskflow/types/latch.py | 7 ++++++- taskflow/utils/lock_utils.py | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/taskflow/types/latch.py b/taskflow/types/latch.py index 9aa2622d..0945a286 100644 --- a/taskflow/types/latch.py +++ b/taskflow/types/latch.py @@ -20,7 +20,12 @@ from taskflow.types import timing as tt class Latch(object): - """A class that ensures N-arrivals occur before unblocking.""" + """A class that ensures N-arrivals occur before unblocking. + + TODO(harlowja): replace with http://bugs.python.org/issue8777 when we no + longer have to support python 2.6 or 2.7 and we can only support 3.2 or + later. + """ def __init__(self, count): count = int(count) diff --git a/taskflow/utils/lock_utils.py b/taskflow/utils/lock_utils.py index dbc0b778..282756f3 100644 --- a/taskflow/utils/lock_utils.py +++ b/taskflow/utils/lock_utils.py @@ -142,6 +142,9 @@ class ReaderWriterLock(_ReaderWriterLockBase): the write lock. In the future these restrictions may be relaxed. + + This can be eventually removed if http://bugs.python.org/issue8800 ever + gets accepted into the python standard threading library... """ WRITER = 'w' READER = 'r' From c5c22112a229cf7705aa23fe754d8f6b4ecb0bec Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 16 Jul 2014 12:21:48 -0700 Subject: [PATCH 030/240] Ensure state machine can be frozen To match the other types ability to be frozen so that they can no longer be mutated add a freeze() method to the state machine type that ensures that subsequent add_state, add_reaction, add_transition method calls will raise an exception. This is quite useful when the state machine is constructed in one function and the creator wants to stop further adds by other functions. To start use this freeze() capability in the runner state machine when a machine build is requested. Part of blueprint runner-state-machine Change-Id: I61488e4158b38d39017435af008382f28d800049 --- taskflow/engines/action_engine/runner.py | 1 + taskflow/tests/unit/test_types.py | 9 +++++++++ taskflow/types/fsm.py | 18 ++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/taskflow/engines/action_engine/runner.py b/taskflow/engines/action_engine/runner.py index 7a0b9c87..e3a6f05c 100644 --- a/taskflow/engines/action_engine/runner.py +++ b/taskflow/engines/action_engine/runner.py @@ -196,6 +196,7 @@ class _MachineBuilder(object): m.add_reaction(st.SCHEDULING, 'schedule', schedule) m.add_reaction(st.WAITING, 'wait', wait) + m.freeze() return (m, memory) diff --git a/taskflow/tests/unit/test_types.py b/taskflow/tests/unit/test_types.py index 141cdfc8..d7198b16 100644 --- a/taskflow/tests/unit/test_types.py +++ b/taskflow/tests/unit/test_types.py @@ -298,6 +298,15 @@ class FSMTest(test.TestCase): self.assertIn(('up', 'fall', 'down'), transitions) self.assertIn(('down', 'jump', 'up'), transitions) + def test_freeze(self): + self.jumper.freeze() + self.assertRaises(fsm.FrozenMachine, self.jumper.add_state, 'test') + self.assertRaises(fsm.FrozenMachine, + self.jumper.add_transition, 'test', 'test', 'test') + self.assertRaises(fsm.FrozenMachine, + self.jumper.add_reaction, + 'test', 'test', lambda *args: 'test') + def test_invalid_callbacks(self): m = fsm.FSM('working') m.add_state('working') diff --git a/taskflow/types/fsm.py b/taskflow/types/fsm.py index cbe85b78..9c24166e 100644 --- a/taskflow/types/fsm.py +++ b/taskflow/types/fsm.py @@ -33,6 +33,12 @@ class _Jump(object): self.on_exit = on_exit +class FrozenMachine(Exception): + """Exception raised when a frozen machine is modified.""" + def __init__(self): + super(FrozenMachine, self).__init__("Frozen machine can't be modified") + + class NotInitialized(excp.TaskFlowException): """Error raised when an action is attempted on a not inited machine.""" @@ -62,6 +68,7 @@ class FSM(object): self._states = OrderedDict() self._start_state = start_state self._current = None + self.frozen = False @property def start_state(self): @@ -89,6 +96,8 @@ class FSM(object): parameter which is the event that is being processed that caused the state transition. """ + if self.frozen: + raise FrozenMachine() if state in self._states: raise excp.Duplicate("State '%s' already defined" % state) if on_enter is not None: @@ -123,6 +132,8 @@ class FSM(object): this process typically repeats) until the state machine reaches a terminal state. """ + if self.frozen: + raise FrozenMachine() if state not in self._states: raise excp.NotFound("Can not add a reaction to event '%s' for an" " undefined state '%s'" % (event, state)) @@ -135,6 +146,8 @@ class FSM(object): def add_transition(self, start, end, event): """Adds an allowed transition from start -> end for the given event.""" + if self.frozen: + raise FrozenMachine() if start not in self._states: raise excp.NotFound("Can not add a transition on event '%s' that" " starts in a undefined state '%s'" % (event, @@ -220,8 +233,13 @@ class FSM(object): event = cb(old_state, new_state, event, *args, **kwargs) def __contains__(self, state): + """Returns if this state exists in the machines known states.""" return state in self._states + def freeze(self): + """Freezes & stops addition of states, transitions, reactions...""" + self.frozen = True + @property def states(self): """Returns the state names.""" From 8bbc2fd05cd6b3c22d9ea96a1e371f1da35db9f5 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 16 Jul 2014 12:33:48 -0700 Subject: [PATCH 031/240] Better handle the tree freeze method Instead of dynamically replacing the existing method with a new method, just have the existing method check if the node has been frozen and immediately abort, this workers better with decorators, subclassing... This also unifies how freezing is done across all types that support it, ensuring that the frozen attribute can be set by users (if they so choose). Change-Id: I1e6c6568b7f91765d654d25ca6e68e9b568603fc --- taskflow/types/tree.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/taskflow/types/tree.py b/taskflow/types/tree.py index 41369b04..6ccd07a6 100644 --- a/taskflow/types/tree.py +++ b/taskflow/types/tree.py @@ -22,6 +22,9 @@ import six class FrozenNode(Exception): """Exception raised when a frozen node is modified.""" + def __init__(self): + super(FrozenNode, self).__init__("Frozen node(s) can't be modified") + class _DFSIter(object): """Depth first iterator (non-recursive) over the child nodes.""" @@ -53,20 +56,22 @@ class Node(object): self.item = item self.parent = None self.metadata = dict(kwargs) + self.frozen = False self._children = [] - self._frozen = False - - def _frozen_add(self, child): - raise FrozenNode("Frozen node(s) can't be modified") def freeze(self): - if not self._frozen: + if not self.frozen: + # This will DFS until all children are frozen as well, only + # after that works do we freeze ourselves (this makes it so + # that we don't become frozen if a child node fails to perform + # the freeze operation). for n in self: n.freeze() - self.add = self._frozen_add - self._frozen = True + self.frozen = True def add(self, child): + if self.frozen: + raise FrozenNode() child.parent = self self._children.append(child) From 34d8d54b0fb8840d2f9e29d38eff0db564bf53e7 Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Fri, 19 Sep 2014 08:52:11 +0000 Subject: [PATCH 032/240] Updated from global requirements Change-Id: I966ff8ba796774c6ebf5b99ae799bb14196fcf81 --- requirements-py3.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index 63880b31..670fb62b 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -11,7 +11,7 @@ six>=1.7.0 networkx>=1.8 Babel>=1.3 # Used for backend storage engine loading. -stevedore>=0.14 +stevedore>=1.0.0 # Apache-2.0 # Used for structured input validation jsonschema>=2.0.0,<3.0.0 # For pretty printing state-machine tables diff --git a/test-requirements.txt b/test-requirements.txt index 4068d786..bbfe80b2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -12,4 +12,4 @@ testtools>=0.9.34 zake>=0.1 # Apache-2.0 # docs build jobs sphinx>=1.1.2,!=1.2.0,<1.3 -oslosphinx>=2.2.0.0a2 +oslosphinx>=2.2.0 # Apache-2.0 From 7fe2f5108cf64d7f7a6ab2229159782398db3200 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 19 Sep 2014 22:13:09 -0700 Subject: [PATCH 033/240] Remove no longer needed r/w lock interface base class This interface/class was put in place when this code was being more actively developed to ensure that both the dummy and the actual class had the same methods, since these two classes have apis that are now stable we no longer need to have a base class to enforce the implemented api that itself provides no added functionality. Change-Id: Ida4f86b308941ff708db5395a1d266c0c4b75815 --- taskflow/utils/lock_utils.py | 62 +++++++++++------------------------- 1 file changed, 19 insertions(+), 43 deletions(-) diff --git a/taskflow/utils/lock_utils.py b/taskflow/utils/lock_utils.py index dbc0b778..74361ab3 100644 --- a/taskflow/utils/lock_utils.py +++ b/taskflow/utils/lock_utils.py @@ -19,7 +19,6 @@ # pulls in oslo.cfg) and is reduced to only what taskflow currently wants to # use from that code. -import abc import collections import contextlib import errno @@ -91,47 +90,7 @@ def locked(*args, **kwargs): return decorator -@six.add_metaclass(abc.ABCMeta) -class _ReaderWriterLockBase(object): - """Base class for reader/writer lock implementations.""" - - @abc.abstractproperty - def has_pending_writers(self): - """Returns if there are writers waiting to become the *one* writer.""" - - @abc.abstractmethod - def is_writer(self, check_pending=True): - """Returns if the caller is the active writer or a pending writer.""" - - @abc.abstractproperty - def owner(self): - """Returns whether the lock is locked by a writer or reader.""" - - @abc.abstractmethod - def is_reader(self): - """Returns if the caller is one of the readers.""" - - @abc.abstractmethod - def read_lock(self): - """Context manager that grants a read lock. - - Will wait until no active or pending writers. - - Raises a RuntimeError if an active or pending writer tries to acquire - a read lock. - """ - - @abc.abstractmethod - def write_lock(self): - """Context manager that grants a write lock. - - Will wait until no active readers. Blocks readers after acquiring. - - Raises a RuntimeError if an active reader attempts to acquire a lock. - """ - - -class ReaderWriterLock(_ReaderWriterLockBase): +class ReaderWriterLock(object): """A reader/writer lock. This lock allows for simultaneous readers to exist but only one writer @@ -154,6 +113,7 @@ class ReaderWriterLock(_ReaderWriterLockBase): @property def has_pending_writers(self): + """Returns if there are writers waiting to become the *one* writer.""" self._cond.acquire() try: return bool(self._pending_writers) @@ -161,6 +121,7 @@ class ReaderWriterLock(_ReaderWriterLockBase): self._cond.release() def is_writer(self, check_pending=True): + """Returns if the caller is the active writer or a pending writer.""" self._cond.acquire() try: me = tu.get_ident() @@ -175,6 +136,7 @@ class ReaderWriterLock(_ReaderWriterLockBase): @property def owner(self): + """Returns whether the lock is locked by a writer or reader.""" self._cond.acquire() try: if self._writer is not None: @@ -186,6 +148,7 @@ class ReaderWriterLock(_ReaderWriterLockBase): self._cond.release() def is_reader(self): + """Returns if the caller is one of the readers.""" self._cond.acquire() try: return tu.get_ident() in self._readers @@ -194,6 +157,13 @@ class ReaderWriterLock(_ReaderWriterLockBase): @contextlib.contextmanager def read_lock(self): + """Context manager that grants a read lock. + + Will wait until no active or pending writers. + + Raises a RuntimeError if an active or pending writer tries to acquire + a read lock. + """ me = tu.get_ident() if self.is_writer(): raise RuntimeError("Writer %s can not acquire a read lock" @@ -226,6 +196,12 @@ class ReaderWriterLock(_ReaderWriterLockBase): @contextlib.contextmanager def write_lock(self): + """Context manager that grants a write lock. + + Will wait until no active readers. Blocks readers after acquiring. + + Raises a RuntimeError if an active reader attempts to acquire a lock. + """ me = tu.get_ident() if self.is_reader(): raise RuntimeError("Reader %s to writer privilege" @@ -257,7 +233,7 @@ class ReaderWriterLock(_ReaderWriterLockBase): self._cond.release() -class DummyReaderWriterLock(_ReaderWriterLockBase): +class DummyReaderWriterLock(object): """A dummy reader/writer lock. This dummy lock doesn't lock anything but provides the same functions as a From 24c65e65b9ce534d89d5d73cb423ffce6995dba7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 20 Sep 2014 08:29:33 -0700 Subject: [PATCH 034/240] Update the requirements-py2.txt file It appears the openstack requirements proposal bot does not currently update all the right files, this bug is filed @ bug 1371936 so until that is fixed this proposes the same change to the py2 requirements file that just got merged into the py3 file. Change-Id: I03dfc1579f44f930acaf92ef47c13829ffcbf35b --- requirements-py2.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-py2.txt b/requirements-py2.txt index bfb837e4..fae98b85 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -13,7 +13,7 @@ six>=1.7.0 networkx>=1.8 Babel>=1.3 # Used for backend storage engine loading. -stevedore>=0.14 +stevedore>=1.0.0 # Apache-2.0 # Backport for concurrent.futures which exists in 3.2+ futures>=2.1.6 # Used for structured input validation From de652c770aa281214aad7003fc6d4d1a85802e4c Mon Sep 17 00:00:00 2001 From: Rafael Rivero Date: Thu, 18 Sep 2014 16:39:28 -0700 Subject: [PATCH 035/240] Typos "searchs" Misspelling of "searchs" in method index. Change-Id: Ifb17fc1a4b4df2ea7cbc3a49e9815888dd1d7467 --- taskflow/types/tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskflow/types/tree.py b/taskflow/types/tree.py index b6527422..759e0fda 100644 --- a/taskflow/types/tree.py +++ b/taskflow/types/tree.py @@ -168,7 +168,7 @@ class Node(object): yield c def index(self, item): - """Finds the child index of a given item, searchs in added order.""" + """Finds the child index of a given item, searches in added order.""" index_at = None for (i, child) in enumerate(self._children): if child.item == item: From aaa51fd689db73f40088eba7a07ac8a131aad528 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 10 Sep 2014 17:13:08 -0700 Subject: [PATCH 036/240] Switch to using oslo.utils and oslo.serialization Instead of copying modules from the incubator into taskflow we can now directly use these same modules from supported libraries instead so this moves the usage of everything except uuidutils which wasn't moved over to using those newly published libraries. Part of blueprint integrate-and-use-oslo-utils-serialization Change-Id: I1183bda96e1ddb062d9cab91990186f0f56f0a0e --- openstack-common.conf | 8 - requirements-py2.txt | 7 +- requirements-py3.txt | 7 +- taskflow/engines/action_engine/engine.py | 3 +- taskflow/engines/helpers.py | 2 +- taskflow/engines/worker_based/executor.py | 3 +- taskflow/engines/worker_based/protocol.py | 2 +- taskflow/jobs/backends/impl_zookeeper.py | 4 +- taskflow/listeners/base.py | 2 +- taskflow/openstack/common/excutils.py | 113 ----- taskflow/openstack/common/gettextutils.py | 479 ------------------ taskflow/openstack/common/importutils.py | 73 --- taskflow/openstack/common/jsonutils.py | 202 -------- taskflow/openstack/common/network_utils.py | 163 ------ taskflow/openstack/common/strutils.py | 311 ------------ taskflow/openstack/common/timeutils.py | 210 -------- taskflow/persistence/backends/impl_dir.py | 2 +- .../persistence/backends/impl_sqlalchemy.py | 2 +- .../persistence/backends/impl_zookeeper.py | 2 +- .../persistence/backends/sqlalchemy/models.py | 4 +- taskflow/persistence/logbook.py | 2 +- taskflow/tests/unit/jobs/test_zk_job.py | 2 +- taskflow/tests/unit/test_engine_helpers.py | 4 +- .../tests/unit/worker_based/test_executor.py | 2 +- taskflow/utils/misc.py | 6 +- taskflow/utils/persistence_utils.py | 3 +- taskflow/utils/reflection.py | 3 +- 27 files changed, 33 insertions(+), 1588 deletions(-) delete mode 100644 taskflow/openstack/common/excutils.py delete mode 100644 taskflow/openstack/common/gettextutils.py delete mode 100644 taskflow/openstack/common/importutils.py delete mode 100644 taskflow/openstack/common/jsonutils.py delete mode 100644 taskflow/openstack/common/network_utils.py delete mode 100644 taskflow/openstack/common/strutils.py delete mode 100644 taskflow/openstack/common/timeutils.py diff --git a/openstack-common.conf b/openstack-common.conf index 8940a040..9db6be0a 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,16 +1,8 @@ [DEFAULT] # The list of modules to copy from oslo-incubator.git -module=excutils -module=importutils -module=jsonutils -module=strutils -module=timeutils module=uuidutils -module=network_utils - script=tools/run_cross_tests.sh # The base module to hold the copy of openstack.common base=taskflow - diff --git a/requirements-py2.txt b/requirements-py2.txt index bfb837e4..a201e9ab 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -3,18 +3,19 @@ # process, which may cause wedges in the gate later. # Packages needed for using this library. -anyjson>=0.3.3 -iso8601>=0.1.9 + # Only needed on python 2.6 ordereddict # Python 2->3 compatibility library. six>=1.7.0 # Very nice graph library networkx>=1.8 -Babel>=1.3 # Used for backend storage engine loading. stevedore>=0.14 # Backport for concurrent.futures which exists in 3.2+ futures>=2.1.6 # Used for structured input validation jsonschema>=2.0.0,<3.0.0 +# For common utilities +oslo.utils>=0.3.0 +oslo.serialization>=0.1.0 diff --git a/requirements-py3.txt b/requirements-py3.txt index 12f78d43..59ee1d36 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -3,14 +3,15 @@ # process, which may cause wedges in the gate later. # Packages needed for using this library. -anyjson>=0.3.3 -iso8601>=0.1.9 + # Python 2->3 compatibility library. six>=1.7.0 # Very nice graph library networkx>=1.8 -Babel>=1.3 # Used for backend storage engine loading. stevedore>=1.0.0 # Apache-2.0 # Used for structured input validation jsonschema>=2.0.0,<3.0.0 +# For common utilities +oslo.utils>=0.3.0 +oslo.serialization>=0.1.0 diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index a5f587fd..9bf62429 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -17,12 +17,13 @@ import contextlib import threading +from oslo.utils import excutils + from taskflow.engines.action_engine import compiler from taskflow.engines.action_engine import executor from taskflow.engines.action_engine import runtime from taskflow.engines import base from taskflow import exceptions as exc -from taskflow.openstack.common import excutils from taskflow import retry from taskflow import states from taskflow import storage as atom_storage diff --git a/taskflow/engines/helpers.py b/taskflow/engines/helpers.py index c200df8a..bfbaaa53 100644 --- a/taskflow/engines/helpers.py +++ b/taskflow/engines/helpers.py @@ -16,11 +16,11 @@ import contextlib +from oslo.utils import importutils import six import stevedore.driver from taskflow import exceptions as exc -from taskflow.openstack.common import importutils from taskflow.persistence import backends as p_backends from taskflow.utils import misc from taskflow.utils import persistence_utils as p_utils diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index 9ff7078b..35fb0bbb 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -18,12 +18,13 @@ import functools import logging import threading +from oslo.utils import timeutils + from taskflow.engines.action_engine import executor from taskflow.engines.worker_based import cache from taskflow.engines.worker_based import protocol as pr from taskflow.engines.worker_based import proxy from taskflow import exceptions as exc -from taskflow.openstack.common import timeutils from taskflow.types import timing as tt from taskflow.utils import async_utils from taskflow.utils import misc diff --git a/taskflow/engines/worker_based/protocol.py b/taskflow/engines/worker_based/protocol.py index 6e54f9fb..a97240a9 100644 --- a/taskflow/engines/worker_based/protocol.py +++ b/taskflow/engines/worker_based/protocol.py @@ -21,11 +21,11 @@ import threading from concurrent import futures import jsonschema from jsonschema import exceptions as schema_exc +from oslo.utils import timeutils import six from taskflow.engines.action_engine import executor from taskflow import exceptions as excp -from taskflow.openstack.common import timeutils from taskflow.types import timing as tt from taskflow.utils import lock_utils from taskflow.utils import misc diff --git a/taskflow/jobs/backends/impl_zookeeper.py b/taskflow/jobs/backends/impl_zookeeper.py index 4fc7b6eb..cc6101c1 100644 --- a/taskflow/jobs/backends/impl_zookeeper.py +++ b/taskflow/jobs/backends/impl_zookeeper.py @@ -24,13 +24,13 @@ from concurrent import futures from kazoo import exceptions as k_exceptions from kazoo.protocol import paths as k_paths from kazoo.recipe import watchers +from oslo.serialization import jsonutils +from oslo.utils import excutils import six from taskflow import exceptions as excp from taskflow.jobs import job as base_job from taskflow.jobs import jobboard -from taskflow.openstack.common import excutils -from taskflow.openstack.common import jsonutils from taskflow.openstack.common import uuidutils from taskflow import states from taskflow.types import timing as tt diff --git a/taskflow/listeners/base.py b/taskflow/listeners/base.py index 352b652a..0b15cce4 100644 --- a/taskflow/listeners/base.py +++ b/taskflow/listeners/base.py @@ -19,9 +19,9 @@ from __future__ import absolute_import import abc import logging +from oslo.utils import excutils import six -from taskflow.openstack.common import excutils from taskflow import states from taskflow.utils import misc diff --git a/taskflow/openstack/common/excutils.py b/taskflow/openstack/common/excutils.py deleted file mode 100644 index 790fc0b1..00000000 --- a/taskflow/openstack/common/excutils.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# Copyright 2012, Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Exception related utilities. -""" - -import logging -import sys -import time -import traceback - -import six - -from taskflow.openstack.common.gettextutils import _LE - - -class save_and_reraise_exception(object): - """Save current exception, run some code and then re-raise. - - In some cases the exception context can be cleared, resulting in None - being attempted to be re-raised after an exception handler is run. This - can happen when eventlet switches greenthreads or when running an - exception handler, code raises and catches an exception. In both - cases the exception context will be cleared. - - To work around this, we save the exception state, run handler code, and - then re-raise the original exception. If another exception occurs, the - saved exception is logged and the new exception is re-raised. - - In some cases the caller may not want to re-raise the exception, and - for those circumstances this context provides a reraise flag that - can be used to suppress the exception. For example:: - - except Exception: - with save_and_reraise_exception() as ctxt: - decide_if_need_reraise() - if not should_be_reraised: - ctxt.reraise = False - - If another exception occurs and reraise flag is False, - the saved exception will not be logged. - - If the caller wants to raise new exception during exception handling - he/she sets reraise to False initially with an ability to set it back to - True if needed:: - - except Exception: - with save_and_reraise_exception(reraise=False) as ctxt: - [if statements to determine whether to raise a new exception] - # Not raising a new exception, so reraise - ctxt.reraise = True - """ - def __init__(self, reraise=True): - self.reraise = reraise - - def __enter__(self): - self.type_, self.value, self.tb, = sys.exc_info() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - if exc_type is not None: - if self.reraise: - logging.error(_LE('Original exception being dropped: %s'), - traceback.format_exception(self.type_, - self.value, - self.tb)) - return False - if self.reraise: - six.reraise(self.type_, self.value, self.tb) - - -def forever_retry_uncaught_exceptions(infunc): - def inner_func(*args, **kwargs): - last_log_time = 0 - last_exc_message = None - exc_count = 0 - while True: - try: - return infunc(*args, **kwargs) - except Exception as exc: - this_exc_message = six.u(str(exc)) - if this_exc_message == last_exc_message: - exc_count += 1 - else: - exc_count = 1 - # Do not log any more frequently than once a minute unless - # the exception message changes - cur_time = int(time.time()) - if (cur_time - last_log_time > 60 or - this_exc_message != last_exc_message): - logging.exception( - _LE('Unexpected exception occurred %d time(s)... ' - 'retrying.') % exc_count) - last_log_time = cur_time - last_exc_message = this_exc_message - exc_count = 0 - # This should be a very rare event. In case it isn't, do - # a sleep. - time.sleep(1) - return inner_func diff --git a/taskflow/openstack/common/gettextutils.py b/taskflow/openstack/common/gettextutils.py deleted file mode 100644 index 20fc2543..00000000 --- a/taskflow/openstack/common/gettextutils.py +++ /dev/null @@ -1,479 +0,0 @@ -# Copyright 2012 Red Hat, Inc. -# Copyright 2013 IBM Corp. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -gettext for openstack-common modules. - -Usual usage in an openstack.common module: - - from taskflow.openstack.common.gettextutils import _ -""" - -import copy -import gettext -import locale -from logging import handlers -import os - -from babel import localedata -import six - -_AVAILABLE_LANGUAGES = {} - -# FIXME(dhellmann): Remove this when moving to oslo.i18n. -USE_LAZY = False - - -class TranslatorFactory(object): - """Create translator functions - """ - - def __init__(self, domain, localedir=None): - """Establish a set of translation functions for the domain. - - :param domain: Name of translation domain, - specifying a message catalog. - :type domain: str - :param lazy: Delays translation until a message is emitted. - Defaults to False. - :type lazy: Boolean - :param localedir: Directory with translation catalogs. - :type localedir: str - """ - self.domain = domain - if localedir is None: - localedir = os.environ.get(domain.upper() + '_LOCALEDIR') - self.localedir = localedir - - def _make_translation_func(self, domain=None): - """Return a new translation function ready for use. - - Takes into account whether or not lazy translation is being - done. - - The domain can be specified to override the default from the - factory, but the localedir from the factory is always used - because we assume the log-level translation catalogs are - installed in the same directory as the main application - catalog. - - """ - if domain is None: - domain = self.domain - t = gettext.translation(domain, - localedir=self.localedir, - fallback=True) - # Use the appropriate method of the translation object based - # on the python version. - m = t.gettext if six.PY3 else t.ugettext - - def f(msg): - """oslo.i18n.gettextutils translation function.""" - if USE_LAZY: - return Message(msg, domain=domain) - return m(msg) - return f - - @property - def primary(self): - "The default translation function." - return self._make_translation_func() - - def _make_log_translation_func(self, level): - return self._make_translation_func(self.domain + '-log-' + level) - - @property - def log_info(self): - "Translate info-level log messages." - return self._make_log_translation_func('info') - - @property - def log_warning(self): - "Translate warning-level log messages." - return self._make_log_translation_func('warning') - - @property - def log_error(self): - "Translate error-level log messages." - return self._make_log_translation_func('error') - - @property - def log_critical(self): - "Translate critical-level log messages." - return self._make_log_translation_func('critical') - - -# NOTE(dhellmann): When this module moves out of the incubator into -# oslo.i18n, these global variables can be moved to an integration -# module within each application. - -# Create the global translation functions. -_translators = TranslatorFactory('taskflow') - -# The primary translation function using the well-known name "_" -_ = _translators.primary - -# Translators for log levels. -# -# The abbreviated names are meant to reflect the usual use of a short -# name like '_'. The "L" is for "log" and the other letter comes from -# the level. -_LI = _translators.log_info -_LW = _translators.log_warning -_LE = _translators.log_error -_LC = _translators.log_critical - -# NOTE(dhellmann): End of globals that will move to the application's -# integration module. - - -def enable_lazy(): - """Convenience function for configuring _() to use lazy gettext - - Call this at the start of execution to enable the gettextutils._ - function to use lazy gettext functionality. This is useful if - your project is importing _ directly instead of using the - gettextutils.install() way of importing the _ function. - """ - global USE_LAZY - USE_LAZY = True - - -def install(domain): - """Install a _() function using the given translation domain. - - Given a translation domain, install a _() function using gettext's - install() function. - - The main difference from gettext.install() is that we allow - overriding the default localedir (e.g. /usr/share/locale) using - a translation-domain-specific environment variable (e.g. - NOVA_LOCALEDIR). - - Note that to enable lazy translation, enable_lazy must be - called. - - :param domain: the translation domain - """ - from six import moves - tf = TranslatorFactory(domain) - moves.builtins.__dict__['_'] = tf.primary - - -class Message(six.text_type): - """A Message object is a unicode object that can be translated. - - Translation of Message is done explicitly using the translate() method. - For all non-translation intents and purposes, a Message is simply unicode, - and can be treated as such. - """ - - def __new__(cls, msgid, msgtext=None, params=None, - domain='taskflow', *args): - """Create a new Message object. - - In order for translation to work gettext requires a message ID, this - msgid will be used as the base unicode text. It is also possible - for the msgid and the base unicode text to be different by passing - the msgtext parameter. - """ - # If the base msgtext is not given, we use the default translation - # of the msgid (which is in English) just in case the system locale is - # not English, so that the base text will be in that locale by default. - if not msgtext: - msgtext = Message._translate_msgid(msgid, domain) - # We want to initialize the parent unicode with the actual object that - # would have been plain unicode if 'Message' was not enabled. - msg = super(Message, cls).__new__(cls, msgtext) - msg.msgid = msgid - msg.domain = domain - msg.params = params - return msg - - def translate(self, desired_locale=None): - """Translate this message to the desired locale. - - :param desired_locale: The desired locale to translate the message to, - if no locale is provided the message will be - translated to the system's default locale. - - :returns: the translated message in unicode - """ - - translated_message = Message._translate_msgid(self.msgid, - self.domain, - desired_locale) - if self.params is None: - # No need for more translation - return translated_message - - # This Message object may have been formatted with one or more - # Message objects as substitution arguments, given either as a single - # argument, part of a tuple, or as one or more values in a dictionary. - # When translating this Message we need to translate those Messages too - translated_params = _translate_args(self.params, desired_locale) - - translated_message = translated_message % translated_params - - return translated_message - - @staticmethod - def _translate_msgid(msgid, domain, desired_locale=None): - if not desired_locale: - system_locale = locale.getdefaultlocale() - # If the system locale is not available to the runtime use English - if not system_locale[0]: - desired_locale = 'en_US' - else: - desired_locale = system_locale[0] - - locale_dir = os.environ.get(domain.upper() + '_LOCALEDIR') - lang = gettext.translation(domain, - localedir=locale_dir, - languages=[desired_locale], - fallback=True) - if six.PY3: - translator = lang.gettext - else: - translator = lang.ugettext - - translated_message = translator(msgid) - return translated_message - - def __mod__(self, other): - # When we mod a Message we want the actual operation to be performed - # by the parent class (i.e. unicode()), the only thing we do here is - # save the original msgid and the parameters in case of a translation - params = self._sanitize_mod_params(other) - unicode_mod = super(Message, self).__mod__(params) - modded = Message(self.msgid, - msgtext=unicode_mod, - params=params, - domain=self.domain) - return modded - - def _sanitize_mod_params(self, other): - """Sanitize the object being modded with this Message. - - - Add support for modding 'None' so translation supports it - - Trim the modded object, which can be a large dictionary, to only - those keys that would actually be used in a translation - - Snapshot the object being modded, in case the message is - translated, it will be used as it was when the Message was created - """ - if other is None: - params = (other,) - elif isinstance(other, dict): - # Merge the dictionaries - # Copy each item in case one does not support deep copy. - params = {} - if isinstance(self.params, dict): - for key, val in self.params.items(): - params[key] = self._copy_param(val) - for key, val in other.items(): - params[key] = self._copy_param(val) - else: - params = self._copy_param(other) - return params - - def _copy_param(self, param): - try: - return copy.deepcopy(param) - except Exception: - # Fallback to casting to unicode this will handle the - # python code-like objects that can't be deep-copied - return six.text_type(param) - - def __add__(self, other): - msg = _('Message objects do not support addition.') - raise TypeError(msg) - - def __radd__(self, other): - return self.__add__(other) - - if six.PY2: - def __str__(self): - # NOTE(luisg): Logging in python 2.6 tries to str() log records, - # and it expects specifically a UnicodeError in order to proceed. - msg = _('Message objects do not support str() because they may ' - 'contain non-ascii characters. ' - 'Please use unicode() or translate() instead.') - raise UnicodeError(msg) - - -def get_available_languages(domain): - """Lists the available languages for the given translation domain. - - :param domain: the domain to get languages for - """ - if domain in _AVAILABLE_LANGUAGES: - return copy.copy(_AVAILABLE_LANGUAGES[domain]) - - localedir = '%s_LOCALEDIR' % domain.upper() - find = lambda x: gettext.find(domain, - localedir=os.environ.get(localedir), - languages=[x]) - - # NOTE(mrodden): en_US should always be available (and first in case - # order matters) since our in-line message strings are en_US - language_list = ['en_US'] - # NOTE(luisg): Babel <1.0 used a function called list(), which was - # renamed to locale_identifiers() in >=1.0, the requirements master list - # requires >=0.9.6, uncapped, so defensively work with both. We can remove - # this check when the master list updates to >=1.0, and update all projects - list_identifiers = (getattr(localedata, 'list', None) or - getattr(localedata, 'locale_identifiers')) - locale_identifiers = list_identifiers() - - for i in locale_identifiers: - if find(i) is not None: - language_list.append(i) - - # NOTE(luisg): Babel>=1.0,<1.3 has a bug where some OpenStack supported - # locales (e.g. 'zh_CN', and 'zh_TW') aren't supported even though they - # are perfectly legitimate locales: - # https://github.com/mitsuhiko/babel/issues/37 - # In Babel 1.3 they fixed the bug and they support these locales, but - # they are still not explicitly "listed" by locale_identifiers(). - # That is why we add the locales here explicitly if necessary so that - # they are listed as supported. - aliases = {'zh': 'zh_CN', - 'zh_Hant_HK': 'zh_HK', - 'zh_Hant': 'zh_TW', - 'fil': 'tl_PH'} - for (locale_, alias) in six.iteritems(aliases): - if locale_ in language_list and alias not in language_list: - language_list.append(alias) - - _AVAILABLE_LANGUAGES[domain] = language_list - return copy.copy(language_list) - - -def translate(obj, desired_locale=None): - """Gets the translated unicode representation of the given object. - - If the object is not translatable it is returned as-is. - If the locale is None the object is translated to the system locale. - - :param obj: the object to translate - :param desired_locale: the locale to translate the message to, if None the - default system locale will be used - :returns: the translated object in unicode, or the original object if - it could not be translated - """ - message = obj - if not isinstance(message, Message): - # If the object to translate is not already translatable, - # let's first get its unicode representation - message = six.text_type(obj) - if isinstance(message, Message): - # Even after unicoding() we still need to check if we are - # running with translatable unicode before translating - return message.translate(desired_locale) - return obj - - -def _translate_args(args, desired_locale=None): - """Translates all the translatable elements of the given arguments object. - - This method is used for translating the translatable values in method - arguments which include values of tuples or dictionaries. - If the object is not a tuple or a dictionary the object itself is - translated if it is translatable. - - If the locale is None the object is translated to the system locale. - - :param args: the args to translate - :param desired_locale: the locale to translate the args to, if None the - default system locale will be used - :returns: a new args object with the translated contents of the original - """ - if isinstance(args, tuple): - return tuple(translate(v, desired_locale) for v in args) - if isinstance(args, dict): - translated_dict = {} - for (k, v) in six.iteritems(args): - translated_v = translate(v, desired_locale) - translated_dict[k] = translated_v - return translated_dict - return translate(args, desired_locale) - - -class TranslationHandler(handlers.MemoryHandler): - """Handler that translates records before logging them. - - The TranslationHandler takes a locale and a target logging.Handler object - to forward LogRecord objects to after translating them. This handler - depends on Message objects being logged, instead of regular strings. - - The handler can be configured declaratively in the logging.conf as follows: - - [handlers] - keys = translatedlog, translator - - [handler_translatedlog] - class = handlers.WatchedFileHandler - args = ('/var/log/api-localized.log',) - formatter = context - - [handler_translator] - class = openstack.common.log.TranslationHandler - target = translatedlog - args = ('zh_CN',) - - If the specified locale is not available in the system, the handler will - log in the default locale. - """ - - def __init__(self, locale=None, target=None): - """Initialize a TranslationHandler - - :param locale: locale to use for translating messages - :param target: logging.Handler object to forward - LogRecord objects to after translation - """ - # NOTE(luisg): In order to allow this handler to be a wrapper for - # other handlers, such as a FileHandler, and still be able to - # configure it using logging.conf, this handler has to extend - # MemoryHandler because only the MemoryHandlers' logging.conf - # parsing is implemented such that it accepts a target handler. - handlers.MemoryHandler.__init__(self, capacity=0, target=target) - self.locale = locale - - def setFormatter(self, fmt): - self.target.setFormatter(fmt) - - def emit(self, record): - # We save the message from the original record to restore it - # after translation, so other handlers are not affected by this - original_msg = record.msg - original_args = record.args - - try: - self._translate_and_log_record(record) - finally: - record.msg = original_msg - record.args = original_args - - def _translate_and_log_record(self, record): - record.msg = translate(record.msg, self.locale) - - # In addition to translating the message, we also need to translate - # arguments that were passed to the log method that were not part - # of the main message e.g., log.info(_('Some message %s'), this_one)) - record.args = _translate_args(record.args, self.locale) - - self.target.emit(record) diff --git a/taskflow/openstack/common/importutils.py b/taskflow/openstack/common/importutils.py deleted file mode 100644 index 1e0e703f..00000000 --- a/taskflow/openstack/common/importutils.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Import related utilities and helper functions. -""" - -import sys -import traceback - - -def import_class(import_str): - """Returns a class from a string including module and class.""" - mod_str, _sep, class_str = import_str.rpartition('.') - __import__(mod_str) - try: - return getattr(sys.modules[mod_str], class_str) - except AttributeError: - raise ImportError('Class %s cannot be found (%s)' % - (class_str, - traceback.format_exception(*sys.exc_info()))) - - -def import_object(import_str, *args, **kwargs): - """Import a class and return an instance of it.""" - return import_class(import_str)(*args, **kwargs) - - -def import_object_ns(name_space, import_str, *args, **kwargs): - """Tries to import object from default namespace. - - Imports a class and return an instance of it, first by trying - to find the class in a default namespace, then failing back to - a full path if not found in the default namespace. - """ - import_value = "%s.%s" % (name_space, import_str) - try: - return import_class(import_value)(*args, **kwargs) - except ImportError: - return import_class(import_str)(*args, **kwargs) - - -def import_module(import_str): - """Import a module.""" - __import__(import_str) - return sys.modules[import_str] - - -def import_versioned_module(version, submodule=None): - module = 'taskflow.v%s' % version - if submodule: - module = '.'.join((module, submodule)) - return import_module(module) - - -def try_import(import_str, default=None): - """Try to import a module and if it fails return default.""" - try: - return import_module(import_str) - except ImportError: - return default diff --git a/taskflow/openstack/common/jsonutils.py b/taskflow/openstack/common/jsonutils.py deleted file mode 100644 index 8231688c..00000000 --- a/taskflow/openstack/common/jsonutils.py +++ /dev/null @@ -1,202 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2011 Justin Santa Barbara -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -''' -JSON related utilities. - -This module provides a few things: - - 1) A handy function for getting an object down to something that can be - JSON serialized. See to_primitive(). - - 2) Wrappers around loads() and dumps(). The dumps() wrapper will - automatically use to_primitive() for you if needed. - - 3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson - is available. -''' - - -import codecs -import datetime -import functools -import inspect -import itertools -import sys - -is_simplejson = False -if sys.version_info < (2, 7): - # On Python <= 2.6, json module is not C boosted, so try to use - # simplejson module if available - try: - import simplejson as json - # NOTE(mriedem): Make sure we have a new enough version of simplejson - # to support the namedobject_as_tuple argument. This can be removed - # in the Kilo release when python 2.6 support is dropped. - if 'namedtuple_as_object' in inspect.getargspec(json.dumps).args: - is_simplejson = True - else: - import json - except ImportError: - import json -else: - import json - -import six -import six.moves.xmlrpc_client as xmlrpclib - -from taskflow.openstack.common import gettextutils -from taskflow.openstack.common import importutils -from taskflow.openstack.common import strutils -from taskflow.openstack.common import timeutils - -netaddr = importutils.try_import("netaddr") - -_nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod, - inspect.isfunction, inspect.isgeneratorfunction, - inspect.isgenerator, inspect.istraceback, inspect.isframe, - inspect.iscode, inspect.isbuiltin, inspect.isroutine, - inspect.isabstract] - -_simple_types = (six.string_types + six.integer_types - + (type(None), bool, float)) - - -def to_primitive(value, convert_instances=False, convert_datetime=True, - level=0, max_depth=3): - """Convert a complex object into primitives. - - Handy for JSON serialization. We can optionally handle instances, - but since this is a recursive function, we could have cyclical - data structures. - - To handle cyclical data structures we could track the actual objects - visited in a set, but not all objects are hashable. Instead we just - track the depth of the object inspections and don't go too deep. - - Therefore, convert_instances=True is lossy ... be aware. - - """ - # handle obvious types first - order of basic types determined by running - # full tests on nova project, resulting in the following counts: - # 572754 - # 460353 - # 379632 - # 274610 - # 199918 - # 114200 - # 51817 - # 26164 - # 6491 - # 283 - # 19 - if isinstance(value, _simple_types): - return value - - if isinstance(value, datetime.datetime): - if convert_datetime: - return timeutils.strtime(value) - else: - return value - - # value of itertools.count doesn't get caught by nasty_type_tests - # and results in infinite loop when list(value) is called. - if type(value) == itertools.count: - return six.text_type(value) - - # FIXME(vish): Workaround for LP bug 852095. Without this workaround, - # tests that raise an exception in a mocked method that - # has a @wrap_exception with a notifier will fail. If - # we up the dependency to 0.5.4 (when it is released) we - # can remove this workaround. - if getattr(value, '__module__', None) == 'mox': - return 'mock' - - if level > max_depth: - return '?' - - # The try block may not be necessary after the class check above, - # but just in case ... - try: - recursive = functools.partial(to_primitive, - convert_instances=convert_instances, - convert_datetime=convert_datetime, - level=level, - max_depth=max_depth) - if isinstance(value, dict): - return dict((k, recursive(v)) for k, v in six.iteritems(value)) - elif isinstance(value, (list, tuple)): - return [recursive(lv) for lv in value] - - # It's not clear why xmlrpclib created their own DateTime type, but - # for our purposes, make it a datetime type which is explicitly - # handled - if isinstance(value, xmlrpclib.DateTime): - value = datetime.datetime(*tuple(value.timetuple())[:6]) - - if convert_datetime and isinstance(value, datetime.datetime): - return timeutils.strtime(value) - elif isinstance(value, gettextutils.Message): - return value.data - elif hasattr(value, 'iteritems'): - return recursive(dict(value.iteritems()), level=level + 1) - elif hasattr(value, '__iter__'): - return recursive(list(value)) - elif convert_instances and hasattr(value, '__dict__'): - # Likely an instance of something. Watch for cycles. - # Ignore class member vars. - return recursive(value.__dict__, level=level + 1) - elif netaddr and isinstance(value, netaddr.IPAddress): - return six.text_type(value) - else: - if any(test(value) for test in _nasty_type_tests): - return six.text_type(value) - return value - except TypeError: - # Class objects are tricky since they may define something like - # __iter__ defined but it isn't callable as list(). - return six.text_type(value) - - -def dumps(value, default=to_primitive, **kwargs): - if is_simplejson: - kwargs['namedtuple_as_object'] = False - return json.dumps(value, default=default, **kwargs) - - -def dump(obj, fp, *args, **kwargs): - if is_simplejson: - kwargs['namedtuple_as_object'] = False - return json.dump(obj, fp, *args, **kwargs) - - -def loads(s, encoding='utf-8', **kwargs): - return json.loads(strutils.safe_decode(s, encoding), **kwargs) - - -def load(fp, encoding='utf-8', **kwargs): - return json.load(codecs.getreader(encoding)(fp), **kwargs) - - -try: - import anyjson -except ImportError: - pass -else: - anyjson._modules.append((__name__, 'dumps', TypeError, - 'loads', ValueError, 'load')) - anyjson.force_implementation(__name__) diff --git a/taskflow/openstack/common/network_utils.py b/taskflow/openstack/common/network_utils.py deleted file mode 100644 index 2729c3fb..00000000 --- a/taskflow/openstack/common/network_utils.py +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2012 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Network-related utilities and helper functions. -""" - -import logging -import socket - -from six.moves.urllib import parse - -from taskflow.openstack.common.gettextutils import _LW - -LOG = logging.getLogger(__name__) - - -def parse_host_port(address, default_port=None): - """Interpret a string as a host:port pair. - - An IPv6 address MUST be escaped if accompanied by a port, - because otherwise ambiguity ensues: 2001:db8:85a3::8a2e:370:7334 - means both [2001:db8:85a3::8a2e:370:7334] and - [2001:db8:85a3::8a2e:370]:7334. - - >>> parse_host_port('server01:80') - ('server01', 80) - >>> parse_host_port('server01') - ('server01', None) - >>> parse_host_port('server01', default_port=1234) - ('server01', 1234) - >>> parse_host_port('[::1]:80') - ('::1', 80) - >>> parse_host_port('[::1]') - ('::1', None) - >>> parse_host_port('[::1]', default_port=1234) - ('::1', 1234) - >>> parse_host_port('2001:db8:85a3::8a2e:370:7334', default_port=1234) - ('2001:db8:85a3::8a2e:370:7334', 1234) - >>> parse_host_port(None) - (None, None) - """ - if not address: - return (None, None) - - if address[0] == '[': - # Escaped ipv6 - _host, _port = address[1:].split(']') - host = _host - if ':' in _port: - port = _port.split(':')[1] - else: - port = default_port - else: - if address.count(':') == 1: - host, port = address.split(':') - else: - # 0 means ipv4, >1 means ipv6. - # We prohibit unescaped ipv6 addresses with port. - host = address - port = default_port - - return (host, None if port is None else int(port)) - - -class ModifiedSplitResult(parse.SplitResult): - """Split results class for urlsplit.""" - - # NOTE(dims): The functions below are needed for Python 2.6.x. - # We can remove these when we drop support for 2.6.x. - @property - def hostname(self): - netloc = self.netloc.split('@', 1)[-1] - host, port = parse_host_port(netloc) - return host - - @property - def port(self): - netloc = self.netloc.split('@', 1)[-1] - host, port = parse_host_port(netloc) - return port - - -def urlsplit(url, scheme='', allow_fragments=True): - """Parse a URL using urlparse.urlsplit(), splitting query and fragments. - This function papers over Python issue9374 when needed. - - The parameters are the same as urlparse.urlsplit. - """ - scheme, netloc, path, query, fragment = parse.urlsplit( - url, scheme, allow_fragments) - if allow_fragments and '#' in path: - path, fragment = path.split('#', 1) - if '?' in path: - path, query = path.split('?', 1) - return ModifiedSplitResult(scheme, netloc, - path, query, fragment) - - -def set_tcp_keepalive(sock, tcp_keepalive=True, - tcp_keepidle=None, - tcp_keepalive_interval=None, - tcp_keepalive_count=None): - """Set values for tcp keepalive parameters - - This function configures tcp keepalive parameters if users wish to do - so. - - :param tcp_keepalive: Boolean, turn on or off tcp_keepalive. If users are - not sure, this should be True, and default values will be used. - - :param tcp_keepidle: time to wait before starting to send keepalive probes - :param tcp_keepalive_interval: time between successive probes, once the - initial wait time is over - :param tcp_keepalive_count: number of probes to send before the connection - is killed - """ - - # NOTE(praneshp): Despite keepalive being a tcp concept, the level is - # still SOL_SOCKET. This is a quirk. - if isinstance(tcp_keepalive, bool): - sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, tcp_keepalive) - else: - raise TypeError("tcp_keepalive must be a boolean") - - if not tcp_keepalive: - return - - # These options aren't available in the OS X version of eventlet, - # Idle + Count * Interval effectively gives you the total timeout. - if tcp_keepidle is not None: - if hasattr(socket, 'TCP_KEEPIDLE'): - sock.setsockopt(socket.IPPROTO_TCP, - socket.TCP_KEEPIDLE, - tcp_keepidle) - else: - LOG.warning(_LW('tcp_keepidle not available on your system')) - if tcp_keepalive_interval is not None: - if hasattr(socket, 'TCP_KEEPINTVL'): - sock.setsockopt(socket.IPPROTO_TCP, - socket.TCP_KEEPINTVL, - tcp_keepalive_interval) - else: - LOG.warning(_LW('tcp_keepintvl not available on your system')) - if tcp_keepalive_count is not None: - if hasattr(socket, 'TCP_KEEPCNT'): - sock.setsockopt(socket.IPPROTO_TCP, - socket.TCP_KEEPCNT, - tcp_keepalive_count) - else: - LOG.warning(_LW('tcp_keepknt not available on your system')) diff --git a/taskflow/openstack/common/strutils.py b/taskflow/openstack/common/strutils.py deleted file mode 100644 index 2f0fd659..00000000 --- a/taskflow/openstack/common/strutils.py +++ /dev/null @@ -1,311 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -System-level utilities and helper functions. -""" - -import math -import re -import sys -import unicodedata - -import six - -from taskflow.openstack.common.gettextutils import _ - - -UNIT_PREFIX_EXPONENT = { - 'k': 1, - 'K': 1, - 'Ki': 1, - 'M': 2, - 'Mi': 2, - 'G': 3, - 'Gi': 3, - 'T': 4, - 'Ti': 4, -} -UNIT_SYSTEM_INFO = { - 'IEC': (1024, re.compile(r'(^[-+]?\d*\.?\d+)([KMGT]i?)?(b|bit|B)$')), - 'SI': (1000, re.compile(r'(^[-+]?\d*\.?\d+)([kMGT])?(b|bit|B)$')), -} - -TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes') -FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no') - -SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]") -SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+") - - -# NOTE(flaper87): The following globals are used by `mask_password` -_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password'] - -# NOTE(ldbragst): Let's build a list of regex objects using the list of -# _SANITIZE_KEYS we already have. This way, we only have to add the new key -# to the list of _SANITIZE_KEYS and we can generate regular expressions -# for XML and JSON automatically. -_SANITIZE_PATTERNS_2 = [] -_SANITIZE_PATTERNS_1 = [] - -# NOTE(amrith): Some regular expressions have only one parameter, some -# have two parameters. Use different lists of patterns here. -_FORMAT_PATTERNS_1 = [r'(%(key)s\s*[=]\s*)[^\s^\'^\"]+'] -_FORMAT_PATTERNS_2 = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])', - r'(%(key)s\s+[\"\']).*?([\"\'])', - r'([-]{2}%(key)s\s+)[^\'^\"^=^\s]+([\s]*)', - r'(<%(key)s>).*?()', - r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])', - r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])', - r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?' - '[\'"]).*?([\'"])', - r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)'] - -for key in _SANITIZE_KEYS: - for pattern in _FORMAT_PATTERNS_2: - reg_ex = re.compile(pattern % {'key': key}, re.DOTALL) - _SANITIZE_PATTERNS_2.append(reg_ex) - - for pattern in _FORMAT_PATTERNS_1: - reg_ex = re.compile(pattern % {'key': key}, re.DOTALL) - _SANITIZE_PATTERNS_1.append(reg_ex) - - -def int_from_bool_as_string(subject): - """Interpret a string as a boolean and return either 1 or 0. - - Any string value in: - - ('True', 'true', 'On', 'on', '1') - - is interpreted as a boolean True. - - Useful for JSON-decoded stuff and config file parsing - """ - return bool_from_string(subject) and 1 or 0 - - -def bool_from_string(subject, strict=False, default=False): - """Interpret a string as a boolean. - - A case-insensitive match is performed such that strings matching 't', - 'true', 'on', 'y', 'yes', or '1' are considered True and, when - `strict=False`, anything else returns the value specified by 'default'. - - Useful for JSON-decoded stuff and config file parsing. - - If `strict=True`, unrecognized values, including None, will raise a - ValueError which is useful when parsing values passed in from an API call. - Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'. - """ - if not isinstance(subject, six.string_types): - subject = six.text_type(subject) - - lowered = subject.strip().lower() - - if lowered in TRUE_STRINGS: - return True - elif lowered in FALSE_STRINGS: - return False - elif strict: - acceptable = ', '.join( - "'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS)) - msg = _("Unrecognized value '%(val)s', acceptable values are:" - " %(acceptable)s") % {'val': subject, - 'acceptable': acceptable} - raise ValueError(msg) - else: - return default - - -def safe_decode(text, incoming=None, errors='strict'): - """Decodes incoming text/bytes string using `incoming` if they're not - already unicode. - - :param incoming: Text's current encoding - :param errors: Errors handling policy. See here for valid - values http://docs.python.org/2/library/codecs.html - :returns: text or a unicode `incoming` encoded - representation of it. - :raises TypeError: If text is not an instance of str - """ - if not isinstance(text, (six.string_types, six.binary_type)): - raise TypeError("%s can't be decoded" % type(text)) - - if isinstance(text, six.text_type): - return text - - if not incoming: - incoming = (sys.stdin.encoding or - sys.getdefaultencoding()) - - try: - return text.decode(incoming, errors) - except UnicodeDecodeError: - # Note(flaper87) If we get here, it means that - # sys.stdin.encoding / sys.getdefaultencoding - # didn't return a suitable encoding to decode - # text. This happens mostly when global LANG - # var is not set correctly and there's no - # default encoding. In this case, most likely - # python will use ASCII or ANSI encoders as - # default encodings but they won't be capable - # of decoding non-ASCII characters. - # - # Also, UTF-8 is being used since it's an ASCII - # extension. - return text.decode('utf-8', errors) - - -def safe_encode(text, incoming=None, - encoding='utf-8', errors='strict'): - """Encodes incoming text/bytes string using `encoding`. - - If incoming is not specified, text is expected to be encoded with - current python's default encoding. (`sys.getdefaultencoding`) - - :param incoming: Text's current encoding - :param encoding: Expected encoding for text (Default UTF-8) - :param errors: Errors handling policy. See here for valid - values http://docs.python.org/2/library/codecs.html - :returns: text or a bytestring `encoding` encoded - representation of it. - :raises TypeError: If text is not an instance of str - """ - if not isinstance(text, (six.string_types, six.binary_type)): - raise TypeError("%s can't be encoded" % type(text)) - - if not incoming: - incoming = (sys.stdin.encoding or - sys.getdefaultencoding()) - - if isinstance(text, six.text_type): - return text.encode(encoding, errors) - elif text and encoding != incoming: - # Decode text before encoding it with `encoding` - text = safe_decode(text, incoming, errors) - return text.encode(encoding, errors) - else: - return text - - -def string_to_bytes(text, unit_system='IEC', return_int=False): - """Converts a string into an float representation of bytes. - - The units supported for IEC :: - - Kb(it), Kib(it), Mb(it), Mib(it), Gb(it), Gib(it), Tb(it), Tib(it) - KB, KiB, MB, MiB, GB, GiB, TB, TiB - - The units supported for SI :: - - kb(it), Mb(it), Gb(it), Tb(it) - kB, MB, GB, TB - - Note that the SI unit system does not support capital letter 'K' - - :param text: String input for bytes size conversion. - :param unit_system: Unit system for byte size conversion. - :param return_int: If True, returns integer representation of text - in bytes. (default: decimal) - :returns: Numerical representation of text in bytes. - :raises ValueError: If text has an invalid value. - - """ - try: - base, reg_ex = UNIT_SYSTEM_INFO[unit_system] - except KeyError: - msg = _('Invalid unit system: "%s"') % unit_system - raise ValueError(msg) - match = reg_ex.match(text) - if match: - magnitude = float(match.group(1)) - unit_prefix = match.group(2) - if match.group(3) in ['b', 'bit']: - magnitude /= 8 - else: - msg = _('Invalid string format: %s') % text - raise ValueError(msg) - if not unit_prefix: - res = magnitude - else: - res = magnitude * pow(base, UNIT_PREFIX_EXPONENT[unit_prefix]) - if return_int: - return int(math.ceil(res)) - return res - - -def to_slug(value, incoming=None, errors="strict"): - """Normalize string. - - Convert to lowercase, remove non-word characters, and convert spaces - to hyphens. - - Inspired by Django's `slugify` filter. - - :param value: Text to slugify - :param incoming: Text's current encoding - :param errors: Errors handling policy. See here for valid - values http://docs.python.org/2/library/codecs.html - :returns: slugified unicode representation of `value` - :raises TypeError: If text is not an instance of str - """ - value = safe_decode(value, incoming, errors) - # NOTE(aababilov): no need to use safe_(encode|decode) here: - # encodings are always "ascii", error handling is always "ignore" - # and types are always known (first: unicode; second: str) - value = unicodedata.normalize("NFKD", value).encode( - "ascii", "ignore").decode("ascii") - value = SLUGIFY_STRIP_RE.sub("", value).strip().lower() - return SLUGIFY_HYPHENATE_RE.sub("-", value) - - -def mask_password(message, secret="***"): - """Replace password with 'secret' in message. - - :param message: The string which includes security information. - :param secret: value with which to replace passwords. - :returns: The unicode value of message with the password fields masked. - - For example: - - >>> mask_password("'adminPass' : 'aaaaa'") - "'adminPass' : '***'" - >>> mask_password("'admin_pass' : 'aaaaa'") - "'admin_pass' : '***'" - >>> mask_password('"password" : "aaaaa"') - '"password" : "***"' - >>> mask_password("'original_password' : 'aaaaa'") - "'original_password' : '***'" - >>> mask_password("u'original_password' : u'aaaaa'") - "u'original_password' : u'***'" - """ - message = six.text_type(message) - - # NOTE(ldbragst): Check to see if anything in message contains any key - # specified in _SANITIZE_KEYS, if not then just return the message since - # we don't have to mask any passwords. - if not any(key in message for key in _SANITIZE_KEYS): - return message - - substitute = r'\g<1>' + secret + r'\g<2>' - for pattern in _SANITIZE_PATTERNS_2: - message = re.sub(pattern, substitute, message) - - substitute = r'\g<1>' + secret - for pattern in _SANITIZE_PATTERNS_1: - message = re.sub(pattern, substitute, message) - - return message diff --git a/taskflow/openstack/common/timeutils.py b/taskflow/openstack/common/timeutils.py deleted file mode 100644 index c48da95f..00000000 --- a/taskflow/openstack/common/timeutils.py +++ /dev/null @@ -1,210 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Time related utilities and helper functions. -""" - -import calendar -import datetime -import time - -import iso8601 -import six - - -# ISO 8601 extended time format with microseconds -_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f' -_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S' -PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND - - -def isotime(at=None, subsecond=False): - """Stringify time in ISO 8601 format.""" - if not at: - at = utcnow() - st = at.strftime(_ISO8601_TIME_FORMAT - if not subsecond - else _ISO8601_TIME_FORMAT_SUBSECOND) - tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC' - st += ('Z' if tz == 'UTC' else tz) - return st - - -def parse_isotime(timestr): - """Parse time from ISO 8601 format.""" - try: - return iso8601.parse_date(timestr) - except iso8601.ParseError as e: - raise ValueError(six.text_type(e)) - except TypeError as e: - raise ValueError(six.text_type(e)) - - -def strtime(at=None, fmt=PERFECT_TIME_FORMAT): - """Returns formatted utcnow.""" - if not at: - at = utcnow() - return at.strftime(fmt) - - -def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT): - """Turn a formatted time back into a datetime.""" - return datetime.datetime.strptime(timestr, fmt) - - -def normalize_time(timestamp): - """Normalize time in arbitrary timezone to UTC naive object.""" - offset = timestamp.utcoffset() - if offset is None: - return timestamp - return timestamp.replace(tzinfo=None) - offset - - -def is_older_than(before, seconds): - """Return True if before is older than seconds.""" - if isinstance(before, six.string_types): - before = parse_strtime(before).replace(tzinfo=None) - else: - before = before.replace(tzinfo=None) - - return utcnow() - before > datetime.timedelta(seconds=seconds) - - -def is_newer_than(after, seconds): - """Return True if after is newer than seconds.""" - if isinstance(after, six.string_types): - after = parse_strtime(after).replace(tzinfo=None) - else: - after = after.replace(tzinfo=None) - - return after - utcnow() > datetime.timedelta(seconds=seconds) - - -def utcnow_ts(): - """Timestamp version of our utcnow function.""" - if utcnow.override_time is None: - # NOTE(kgriffs): This is several times faster - # than going through calendar.timegm(...) - return int(time.time()) - - return calendar.timegm(utcnow().timetuple()) - - -def utcnow(): - """Overridable version of utils.utcnow.""" - if utcnow.override_time: - try: - return utcnow.override_time.pop(0) - except AttributeError: - return utcnow.override_time - return datetime.datetime.utcnow() - - -def iso8601_from_timestamp(timestamp): - """Returns an iso8601 formatted date from timestamp.""" - return isotime(datetime.datetime.utcfromtimestamp(timestamp)) - - -utcnow.override_time = None - - -def set_time_override(override_time=None): - """Overrides utils.utcnow. - - Make it return a constant time or a list thereof, one at a time. - - :param override_time: datetime instance or list thereof. If not - given, defaults to the current UTC time. - """ - utcnow.override_time = override_time or datetime.datetime.utcnow() - - -def advance_time_delta(timedelta): - """Advance overridden time using a datetime.timedelta.""" - assert utcnow.override_time is not None - try: - for dt in utcnow.override_time: - dt += timedelta - except TypeError: - utcnow.override_time += timedelta - - -def advance_time_seconds(seconds): - """Advance overridden time by seconds.""" - advance_time_delta(datetime.timedelta(0, seconds)) - - -def clear_time_override(): - """Remove the overridden time.""" - utcnow.override_time = None - - -def marshall_now(now=None): - """Make an rpc-safe datetime with microseconds. - - Note: tzinfo is stripped, but not required for relative times. - """ - if not now: - now = utcnow() - return dict(day=now.day, month=now.month, year=now.year, hour=now.hour, - minute=now.minute, second=now.second, - microsecond=now.microsecond) - - -def unmarshall_time(tyme): - """Unmarshall a datetime dict.""" - return datetime.datetime(day=tyme['day'], - month=tyme['month'], - year=tyme['year'], - hour=tyme['hour'], - minute=tyme['minute'], - second=tyme['second'], - microsecond=tyme['microsecond']) - - -def delta_seconds(before, after): - """Return the difference between two timing objects. - - Compute the difference in seconds between two date, time, or - datetime objects (as a float, to microsecond resolution). - """ - delta = after - before - return total_seconds(delta) - - -def total_seconds(delta): - """Return the total seconds of datetime.timedelta object. - - Compute total seconds of datetime.timedelta, datetime.timedelta - doesn't have method total_seconds in Python2.6, calculate it manually. - """ - try: - return delta.total_seconds() - except AttributeError: - return ((delta.days * 24 * 3600) + delta.seconds + - float(delta.microseconds) / (10 ** 6)) - - -def is_soon(dt, window): - """Determines if time is going to happen in the next window seconds. - - :param dt: the time - :param window: minimum seconds to remain to consider the time not soon - - :return: True if expiration is within the given duration - """ - soon = (utcnow() + datetime.timedelta(seconds=window)) - return normalize_time(dt) <= soon diff --git a/taskflow/persistence/backends/impl_dir.py b/taskflow/persistence/backends/impl_dir.py index 9ce4a324..f469a1db 100644 --- a/taskflow/persistence/backends/impl_dir.py +++ b/taskflow/persistence/backends/impl_dir.py @@ -20,10 +20,10 @@ import logging import os import shutil +from oslo.serialization import jsonutils import six from taskflow import exceptions as exc -from taskflow.openstack.common import jsonutils from taskflow.persistence.backends import base from taskflow.persistence import logbook from taskflow.utils import lock_utils diff --git a/taskflow/persistence/backends/impl_sqlalchemy.py b/taskflow/persistence/backends/impl_sqlalchemy.py index 1dc008eb..587d4d25 100644 --- a/taskflow/persistence/backends/impl_sqlalchemy.py +++ b/taskflow/persistence/backends/impl_sqlalchemy.py @@ -25,6 +25,7 @@ import functools import logging import time +from oslo.utils import strutils import six import sqlalchemy as sa from sqlalchemy import exc as sa_exc @@ -32,7 +33,6 @@ from sqlalchemy import orm as sa_orm from sqlalchemy import pool as sa_pool from taskflow import exceptions as exc -from taskflow.openstack.common import strutils from taskflow.persistence.backends import base from taskflow.persistence.backends.sqlalchemy import migration from taskflow.persistence.backends.sqlalchemy import models diff --git a/taskflow/persistence/backends/impl_zookeeper.py b/taskflow/persistence/backends/impl_zookeeper.py index e60bad85..948c54dc 100644 --- a/taskflow/persistence/backends/impl_zookeeper.py +++ b/taskflow/persistence/backends/impl_zookeeper.py @@ -19,9 +19,9 @@ import logging from kazoo import exceptions as k_exc from kazoo.protocol import paths +from oslo.serialization import jsonutils from taskflow import exceptions as exc -from taskflow.openstack.common import jsonutils from taskflow.persistence.backends import base from taskflow.persistence import logbook from taskflow.utils import kazoo_utils as k_utils diff --git a/taskflow/persistence/backends/sqlalchemy/models.py b/taskflow/persistence/backends/sqlalchemy/models.py index 4a78c5cb..47b8c839 100644 --- a/taskflow/persistence/backends/sqlalchemy/models.py +++ b/taskflow/persistence/backends/sqlalchemy/models.py @@ -15,6 +15,8 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.serialization import jsonutils +from oslo.utils import timeutils from sqlalchemy import Column, String, DateTime, Enum from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import ForeignKey @@ -22,8 +24,6 @@ from sqlalchemy.orm import backref from sqlalchemy.orm import relationship from sqlalchemy import types as types -from taskflow.openstack.common import jsonutils -from taskflow.openstack.common import timeutils from taskflow.openstack.common import uuidutils from taskflow.persistence import logbook from taskflow import states diff --git a/taskflow/persistence/logbook.py b/taskflow/persistence/logbook.py index 12c6c996..974d8461 100644 --- a/taskflow/persistence/logbook.py +++ b/taskflow/persistence/logbook.py @@ -19,10 +19,10 @@ import abc import copy import logging +from oslo.utils import timeutils import six from taskflow import exceptions as exc -from taskflow.openstack.common import timeutils from taskflow.openstack.common import uuidutils from taskflow import states from taskflow.utils import misc diff --git a/taskflow/tests/unit/jobs/test_zk_job.py b/taskflow/tests/unit/jobs/test_zk_job.py index 7268a1a4..5a536f9e 100644 --- a/taskflow/tests/unit/jobs/test_zk_job.py +++ b/taskflow/tests/unit/jobs/test_zk_job.py @@ -14,13 +14,13 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.serialization import jsonutils import six import testtools from zake import fake_client from zake import utils as zake_utils from taskflow.jobs.backends import impl_zookeeper -from taskflow.openstack.common import jsonutils from taskflow.openstack.common import uuidutils from taskflow import states from taskflow import test diff --git a/taskflow/tests/unit/test_engine_helpers.py b/taskflow/tests/unit/test_engine_helpers.py index 30ff51c3..fbf1756c 100644 --- a/taskflow/tests/unit/test_engine_helpers.py +++ b/taskflow/tests/unit/test_engine_helpers.py @@ -69,7 +69,7 @@ class FlowFromDetailTestCase(test.TestCase): _lb, flow_detail = p_utils.temporary_flow_detail() flow_detail.meta = dict(factory=dict(name=name)) - with mock.patch('taskflow.openstack.common.importutils.import_class', + with mock.patch('oslo.utils.importutils.import_class', return_value=lambda: 'RESULT') as mock_import: result = taskflow.engines.flow_from_detail(flow_detail) mock_import.assert_called_onec_with(name) @@ -80,7 +80,7 @@ class FlowFromDetailTestCase(test.TestCase): _lb, flow_detail = p_utils.temporary_flow_detail() flow_detail.meta = dict(factory=dict(name=name, args=['foo'])) - with mock.patch('taskflow.openstack.common.importutils.import_class', + with mock.patch('oslo.utils.importutils.import_class', return_value=lambda x: 'RESULT %s' % x) as mock_import: result = taskflow.engines.flow_from_detail(flow_detail) mock_import.assert_called_onec_with(name) diff --git a/taskflow/tests/unit/worker_based/test_executor.py b/taskflow/tests/unit/worker_based/test_executor.py index e6c97e17..cc184df2 100644 --- a/taskflow/tests/unit/worker_based/test_executor.py +++ b/taskflow/tests/unit/worker_based/test_executor.py @@ -19,10 +19,10 @@ import time from concurrent import futures import mock +from oslo.utils import timeutils from taskflow.engines.worker_based import executor from taskflow.engines.worker_based import protocol as pr -from taskflow.openstack.common import timeutils from taskflow import test from taskflow.tests import utils from taskflow.utils import misc diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 035e86ac..1d7304e7 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -30,12 +30,12 @@ import sys import time import traceback +from oslo.serialization import jsonutils +from oslo.utils import netutils import six from six.moves.urllib import parse as urlparse from taskflow import exceptions as exc -from taskflow.openstack.common import jsonutils -from taskflow.openstack.common import network_utils from taskflow.utils import reflection @@ -82,7 +82,7 @@ def parse_uri(uri, query_duplicates=False): if not match: raise ValueError("Uri %r does not start with a RFC 3986 compliant" " scheme" % (uri)) - parsed = network_utils.urlsplit(uri) + parsed = netutils.urlsplit(uri) if parsed.query: query_params = urlparse.parse_qsl(parsed.query) if not query_duplicates: diff --git a/taskflow/utils/persistence_utils.py b/taskflow/utils/persistence_utils.py index e3c4ba36..dbcdac29 100644 --- a/taskflow/utils/persistence_utils.py +++ b/taskflow/utils/persistence_utils.py @@ -17,7 +17,8 @@ import contextlib import logging -from taskflow.openstack.common import timeutils +from oslo.utils import timeutils + from taskflow.openstack.common import uuidutils from taskflow.persistence import logbook from taskflow.utils import misc diff --git a/taskflow/utils/reflection.py b/taskflow/utils/reflection.py index bc5a3223..34793e00 100644 --- a/taskflow/utils/reflection.py +++ b/taskflow/utils/reflection.py @@ -17,10 +17,9 @@ import inspect import types +from oslo.utils import importutils import six -from taskflow.openstack.common import importutils - try: _TYPE_TYPE = types.TypeType except AttributeError: From 96014cbba3bcb471d36796d2dc9662d22bd89552 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 20 Sep 2014 14:56:26 -0700 Subject: [PATCH 037/240] Color some of the states depending on there meaning Instead of just having black text color adjust some of the states text color depending on there type/name and highlight some as red, green, orange depending on there underlying meaning. Color names are from: http://www.graphviz.org/doc/info/colors.html Change-Id: I89f8f90837551a257936d254516ada6130e7b6da --- doc/source/img/engine_states.svg | 6 +++--- doc/source/img/flow_states.svg | 6 +++--- doc/source/img/retry_states.svg | 6 +++--- doc/source/img/task_states.svg | 6 +++--- tools/state_graph.py | 22 ++++++++++++++++++---- 5 files changed, 30 insertions(+), 16 deletions(-) diff --git a/doc/source/img/engine_states.svg b/doc/source/img/engine_states.svg index 8ef68c3e..079002ec 100644 --- a/doc/source/img/engine_states.svg +++ b/doc/source/img/engine_states.svg @@ -1,8 +1,8 @@ - - -Engines statesGAME_OVERREVERTEDSUCCESSSUSPENDEDFAILUREUNDEFINEDRESUMINGSCHEDULINGANALYZINGWAITINGstart + +Engines statesGAME_OVERREVERTEDSUCCESSSUSPENDEDFAILUREUNDEFINEDRESUMINGSCHEDULINGANALYZINGWAITINGstart diff --git a/doc/source/img/flow_states.svg b/doc/source/img/flow_states.svg index 5a1cdcbd..80bf1a0a 100644 --- a/doc/source/img/flow_states.svg +++ b/doc/source/img/flow_states.svg @@ -1,8 +1,8 @@ - - -Flow statesPENDINGRUNNINGFAILURESUSPENDINGREVERTEDSUCCESSRESUMINGSUSPENDEDstart + +Flow statesPENDINGRUNNINGFAILURESUSPENDINGREVERTEDSUCCESSRESUMINGSUSPENDEDstart diff --git a/doc/source/img/retry_states.svg b/doc/source/img/retry_states.svg index a2ba2fa9..8b0c6357 100644 --- a/doc/source/img/retry_states.svg +++ b/doc/source/img/retry_states.svg @@ -1,8 +1,8 @@ - - -Retries statesPENDINGRUNNINGSUCCESSFAILURERETRYINGREVERTINGREVERTEDstart + +Retries statesPENDINGRUNNINGSUCCESSFAILURERETRYINGREVERTINGREVERTEDstart diff --git a/doc/source/img/task_states.svg b/doc/source/img/task_states.svg index c281be8e..14a1f098 100644 --- a/doc/source/img/task_states.svg +++ b/doc/source/img/task_states.svg @@ -1,8 +1,8 @@ - - -Tasks statesPENDINGRUNNINGSUCCESSFAILUREREVERTINGREVERTEDstart + +Tasks statesPENDINGRUNNINGSUCCESSFAILUREREVERTINGREVERTEDstart diff --git a/tools/state_graph.py b/tools/state_graph.py index e83426bf..c5d72d02 100755 --- a/tools/state_graph.py +++ b/tools/state_graph.py @@ -57,6 +57,18 @@ def make_machine(start_state, transitions, disallowed): return machine +def map_color(internal_states, state): + if state in internal_states: + return 'blue' + if state == states.FAILURE: + return 'red' + if state == states.REVERTED: + return 'darkorange' + if state == states.SUCCESS: + return 'green' + return None + + def main(): parser = optparse.OptionParser() parser.add_option("-f", "--file", dest="filename", @@ -119,14 +131,16 @@ def main(): for (start_state, _on_event, end_state) in source: if start_state not in nodes: start_node_attrs = node_attrs.copy() - if start_state in internal_states: - start_node_attrs['fontcolor'] = 'blue' + text_color = map_color(internal_states, start_state) + if text_color: + start_node_attrs['fontcolor'] = text_color nodes[start_state] = pydot.Node(start_state, **start_node_attrs) g.add_node(nodes[start_state]) if end_state not in nodes: end_node_attrs = node_attrs.copy() - if end_state in internal_states: - end_node_attrs['fontcolor'] = 'blue' + text_color = map_color(internal_states, end_state) + if text_color: + end_node_attrs['fontcolor'] = text_color nodes[end_state] = pydot.Node(end_state, **end_node_attrs) g.add_node(nodes[end_state]) g.add_edge(pydot.Edge(nodes[start_state], nodes[end_state])) From d3d66083f43398a1db6c58fa1c48bb58110e276c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Aug 2014 10:37:57 -0700 Subject: [PATCH 038/240] Increase/adjust the logging of the WBE response/send activities Change-Id: I1d8309ce87114a0890dfc93a0a2c4b68f80ef828 --- taskflow/engines/worker_based/executor.py | 27 +++++++++++++++++------ taskflow/engines/worker_based/proxy.py | 2 +- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index 813612cd..c8857b03 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -98,11 +98,14 @@ class WorkerTaskExecutor(executor.TaskExecutorBase): def _process_notify(self, notify, message): """Process notify message from remote side.""" - LOG.debug("Start processing notify message.") + LOG.debug("Started processing notify message '%s'", + message.delivery_tag) topic = notify['topic'] tasks = notify['tasks'] - # add worker info to the cache + # Add worker info to the cache + LOG.debug("Received that tasks %s can be processed by topic '%s'", + tasks, topic) self._workers_arrival.acquire() try: self._workers_cache[topic] = tasks @@ -110,22 +113,25 @@ class WorkerTaskExecutor(executor.TaskExecutorBase): finally: self._workers_arrival.release() - # publish waiting requests + # Publish waiting requests for request in self._requests_cache.get_waiting_requests(tasks): if request.transition_and_log_error(pr.PENDING, logger=LOG): self._publish_request(request, topic) def _process_response(self, response, message): """Process response from remote side.""" - LOG.debug("Start processing response message.") + LOG.debug("Started processing response message '%s'", + message.delivery_tag) try: task_uuid = message.properties['correlation_id'] except KeyError: - LOG.warning("The 'correlation_id' message property is missing.") + LOG.warning("The 'correlation_id' message property is missing") else: request = self._requests_cache.get(task_uuid) if request is not None: response = pr.Response.from_dict(response) + LOG.debug("Response with state '%s' received for '%s'", + response.state, request) if response.state == pr.RUNNING: request.transition_and_log_error(pr.RUNNING, logger=LOG) elif response.state == pr.PROGRESS: @@ -144,7 +150,7 @@ class WorkerTaskExecutor(executor.TaskExecutorBase): LOG.warning("Unexpected response status: '%s'", response.state) else: - LOG.debug("Request with id='%s' not found.", task_uuid) + LOG.debug("Request with id='%s' not found", task_uuid) @staticmethod def _handle_expired_request(request): @@ -191,12 +197,18 @@ class WorkerTaskExecutor(executor.TaskExecutorBase): self._requests_cache[request.uuid] = request self._publish_request(request, topic) else: + LOG.debug("Delaying submission of '%s', no currently known" + " worker/s available to process it", request) self._requests_cache[request.uuid] = request return request.result def _publish_request(self, request, topic): """Publish request to a given topic.""" + LOG.debug("Submitting execution of '%s' to topic '%s' (expecting" + " response identified by reply_to=%s and" + " correlation_id=%s)", request, topic, self._uuid, + request.uuid) try: self._proxy.publish(msg=request, routing_key=topic, @@ -204,7 +216,8 @@ class WorkerTaskExecutor(executor.TaskExecutorBase): correlation_id=request.uuid) except Exception: with misc.capture_failure() as failure: - LOG.exception("Failed to submit the '%s' request.", request) + LOG.warn("Failed to submit '%s' (transitioning it to" + " %s)", request, pr.FAILURE, exc_info=True) if request.transition_and_log_error(pr.FAILURE, logger=LOG): del self._requests_cache[request.uuid] request.set_result(failure) diff --git a/taskflow/engines/worker_based/proxy.py b/taskflow/engines/worker_based/proxy.py index d2991ca3..c51dd164 100644 --- a/taskflow/engines/worker_based/proxy.py +++ b/taskflow/engines/worker_based/proxy.py @@ -95,11 +95,11 @@ class Proxy(object): def publish(self, msg, routing_key, **kwargs): """Publish message to the named exchange with given routing key.""" - LOG.debug("Sending %s", msg) if isinstance(routing_key, six.string_types): routing_keys = [routing_key] else: routing_keys = routing_key + LOG.debug("Sending '%s' using routing keys %s", msg, routing_keys) with kombu.producers[self._conn].acquire(block=True) as producer: for routing_key in routing_keys: queue = self._make_queue(routing_key, self._exchange) From 83690a20863702c3bcebc042d2edefe7161062a8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 19 Sep 2014 15:41:45 -0700 Subject: [PATCH 039/240] Fix multilock concurrency when shared by > 1 threads Instead of raising thread errors when another thread has locks that the consuming thread wants to use just block and wait and release the correct locks on release to match the expected vs observed behavior. This makes it so that a single multilock object can be shared by many threads and each thread using the object will correctly obtain and release as expected... Fixes bug 1371814 Change-Id: Ia21a05fe9249fa019a09c4f30beeb0770ded5150 --- taskflow/tests/unit/test_utils_lock_utils.py | 218 +++++++++++++++++++ taskflow/utils/lock_utils.py | 124 +++++++++-- 2 files changed, 318 insertions(+), 24 deletions(-) diff --git a/taskflow/tests/unit/test_utils_lock_utils.py b/taskflow/tests/unit/test_utils_lock_utils.py index 2b2f1f83..066b17a2 100644 --- a/taskflow/tests/unit/test_utils_lock_utils.py +++ b/taskflow/tests/unit/test_utils_lock_utils.py @@ -19,6 +19,7 @@ import threading import time from concurrent import futures +import mock from taskflow import test from taskflow.utils import lock_utils @@ -85,6 +86,223 @@ def _spawn_variation(readers, writers, max_workers=None): return (writer_times, reader_times) +class MultilockTest(test.TestCase): + def test_empty_error(self): + self.assertRaises(ValueError, + lock_utils.MultiLock, []) + self.assertRaises(ValueError, + lock_utils.MultiLock, ()) + self.assertRaises(ValueError, + lock_utils.MultiLock, iter([])) + + def test_creation(self): + locks = [] + for _i in range(0, 10): + locks.append(threading.Lock()) + n_lock = lock_utils.MultiLock(locks) + self.assertEqual(0, n_lock.obtained) + self.assertEqual(len(locks), len(n_lock)) + + def test_acquired(self): + lock1 = threading.Lock() + lock2 = threading.Lock() + n_lock = lock_utils.MultiLock((lock1, lock2)) + self.assertTrue(n_lock.acquire()) + try: + self.assertTrue(lock1.locked()) + self.assertTrue(lock2.locked()) + finally: + n_lock.release() + self.assertFalse(lock1.locked()) + self.assertFalse(lock2.locked()) + + def test_acquired_context_manager(self): + lock1 = threading.Lock() + n_lock = lock_utils.MultiLock([lock1]) + with n_lock as gotten: + self.assertTrue(gotten) + self.assertTrue(lock1.locked()) + self.assertFalse(lock1.locked()) + self.assertEqual(0, n_lock.obtained) + + def test_partial_acquired(self): + lock1 = threading.Lock() + lock2 = mock.create_autospec(threading.Lock()) + lock2.acquire.return_value = False + n_lock = lock_utils.MultiLock((lock1, lock2)) + with n_lock as gotten: + self.assertFalse(gotten) + self.assertTrue(lock1.locked()) + self.assertEqual(1, n_lock.obtained) + self.assertEqual(2, len(n_lock)) + self.assertEqual(0, n_lock.obtained) + + def test_partial_acquired_failure(self): + lock1 = threading.Lock() + lock2 = mock.create_autospec(threading.Lock()) + lock2.acquire.side_effect = RuntimeError("Broke") + n_lock = lock_utils.MultiLock((lock1, lock2)) + self.assertRaises(threading.ThreadError, n_lock.acquire) + self.assertEqual(1, n_lock.obtained) + n_lock.release() + + def test_release_failure(self): + lock1 = threading.Lock() + lock2 = mock.create_autospec(threading.Lock()) + lock2.acquire.return_value = True + lock2.release.side_effect = RuntimeError("Broke") + n_lock = lock_utils.MultiLock((lock1, lock2)) + self.assertTrue(n_lock.acquire()) + self.assertEqual(2, n_lock.obtained) + self.assertRaises(threading.ThreadError, n_lock.release) + self.assertEqual(2, n_lock.obtained) + lock2.release.side_effect = None + n_lock.release() + self.assertEqual(0, n_lock.obtained) + + def test_release_partial_failure(self): + lock1 = threading.Lock() + lock2 = mock.create_autospec(threading.Lock()) + lock2.acquire.return_value = True + lock2.release.side_effect = RuntimeError("Broke") + lock3 = threading.Lock() + n_lock = lock_utils.MultiLock((lock1, lock2, lock3)) + self.assertTrue(n_lock.acquire()) + self.assertEqual(3, n_lock.obtained) + self.assertRaises(threading.ThreadError, n_lock.release) + self.assertEqual(2, n_lock.obtained) + lock2.release.side_effect = None + n_lock.release() + self.assertEqual(0, n_lock.obtained) + + def test_acquired_pass(self): + activated = collections.deque() + lock1 = threading.Lock() + lock2 = threading.Lock() + n_lock = lock_utils.MultiLock((lock1, lock2)) + + def critical_section(): + start = time.time() + time.sleep(0.05) + end = time.time() + activated.append((start, end)) + + def run(): + with n_lock: + critical_section() + + threads = [] + for _i in range(0, 20): + t = threading.Thread(target=run) + t.daemon = True + threads.append(t) + t.start() + while threads: + t = threads.pop() + t.join() + for (start, end) in activated: + self.assertEqual(1, _find_overlaps(activated, start, end)) + + self.assertFalse(lock1.locked()) + self.assertFalse(lock2.locked()) + + def test_acquired_fail(self): + activated = collections.deque() + lock1 = threading.Lock() + lock2 = threading.Lock() + n_lock = lock_utils.MultiLock((lock1, lock2)) + + def run(): + with n_lock: + start = time.time() + time.sleep(0.05) + end = time.time() + activated.append((start, end)) + + def run_fail(): + try: + with n_lock: + raise RuntimeError() + except RuntimeError: + pass + + threads = [] + for i in range(0, 20): + if i % 2 == 1: + target = run_fail + else: + target = run + t = threading.Thread(target=target) + threads.append(t) + t.daemon = True + t.start() + while threads: + t = threads.pop() + t.join() + + for (start, end) in activated: + self.assertEqual(1, _find_overlaps(activated, start, end)) + self.assertFalse(lock1.locked()) + self.assertFalse(lock2.locked()) + + def test_double_acquire_single(self): + activated = collections.deque() + + def run(): + start = time.time() + time.sleep(0.05) + end = time.time() + activated.append((start, end)) + + lock1 = threading.RLock() + lock2 = threading.RLock() + n_lock = lock_utils.MultiLock((lock1, lock2)) + with n_lock: + run() + with n_lock: + run() + run() + + for (start, end) in activated: + self.assertEqual(1, _find_overlaps(activated, start, end)) + + def test_double_acquire_many(self): + activated = collections.deque() + n_lock = lock_utils.MultiLock((threading.RLock(), threading.RLock())) + + def critical_section(): + start = time.time() + time.sleep(0.05) + end = time.time() + activated.append((start, end)) + + def run(): + with n_lock: + critical_section() + with n_lock: + critical_section() + critical_section() + + threads = [] + for i in range(0, 20): + t = threading.Thread(target=run) + threads.append(t) + t.daemon = True + t.start() + while threads: + t = threads.pop() + t.join() + + for (start, end) in activated: + self.assertEqual(1, _find_overlaps(activated, start, end)) + + def test_no_acquire_release(self): + lock1 = threading.Lock() + lock2 = threading.Lock() + n_lock = lock_utils.MultiLock((lock1, lock2)) + self.assertRaises(threading.ThreadError, n_lock.release) + + class ReadWriteLockTest(test.TestCase): def test_writer_abort(self): lock = lock_utils.ReaderWriterLock() diff --git a/taskflow/utils/lock_utils.py b/taskflow/utils/lock_utils.py index dbc0b778..dab08537 100644 --- a/taskflow/utils/lock_utils.py +++ b/taskflow/utils/lock_utils.py @@ -291,46 +291,122 @@ class MultiLock(object): """A class which attempts to obtain & release many locks at once. It is typically useful as a context manager around many locks (instead of - having to nest individual lock context managers). + having to nest individual lock context managers, which can become pretty + awkward looking). + + NOTE(harlowja): The locks that will be obtained will be in the order the + locks are given in the constructor, they will be acquired in order and + released in reverse order (so ordering matters). """ def __init__(self, locks): - assert len(locks) > 0, "Zero locks requested" + if not isinstance(locks, tuple): + locks = tuple(locks) + if len(locks) <= 0: + raise ValueError("Zero locks requested") self._locks = locks - self._locked = [False] * len(locks) + self._local = threading.local() + + @property + def _lock_stacks(self): + # This is weird, but this is how thread locals work (in that each + # thread will need to check if it has already created the attribute and + # if not then create it and set it to the thread local variable...) + # + # This isn't done in the constructor since the constructor is only + # activated by one of the many threads that could use this object, + # and that means that the attribute will only exist for that one + # thread. + try: + return self._local.stacks + except AttributeError: + self._local.stacks = [] + return self._local.stacks def __enter__(self): - self.acquire() + return self.acquire() + + @property + def obtained(self): + """Returns how many locks were last acquired/obtained.""" + try: + return self._lock_stacks[-1] + except IndexError: + return 0 + + def __len__(self): + return len(self._locks) def acquire(self): + """This will attempt to acquire all the locks given in the constructor. - def is_locked(lock): - # NOTE(harlowja): reentrant locks (rlock) don't have this - # attribute, but normal non-reentrant locks do, how odd... - if hasattr(lock, 'locked'): - return lock.locked() - return False + If all the locks can not be acquired (and say only X of Y locks could + be acquired then this will return false to signify that not all the + locks were able to be acquired, you can later use the :attr:`.obtained` + property to determine how many were obtained during the last + acquisition attempt). - for i in range(0, len(self._locked)): - if self._locked[i] or is_locked(self._locks[i]): - raise threading.ThreadError("Lock %s not previously released" - % (i + 1)) - self._locked[i] = False - - for (i, lock) in enumerate(self._locks): - self._locked[i] = lock.acquire() + NOTE(harlowja): When not all locks were acquired it is still required + to release since under partial acquisition the acquired locks + must still be released. For example if 4 out of 5 locks were acquired + this will return false, but the user **must** still release those + other 4 to avoid causing locking issues... + """ + gotten = 0 + for lock in self._locks: + try: + acked = lock.acquire() + except (threading.ThreadError, RuntimeError) as e: + # If we have already gotten some set of the desired locks + # make sure we track that and ensure that we later release them + # instead of losing them. + if gotten: + self._lock_stacks.append(gotten) + raise threading.ThreadError( + "Unable to acquire lock %s/%s due to '%s'" + % (gotten + 1, len(self._locks), e)) + else: + if not acked: + break + else: + gotten += 1 + if gotten: + self._lock_stacks.append(gotten) + return gotten == len(self._locks) def __exit__(self, type, value, traceback): self.release() def release(self): - for (i, locked) in enumerate(self._locked): + """Releases any past acquired locks (partial or otherwise).""" + height = len(self._lock_stacks) + if not height: + # Raise the same error type as the threading.Lock raises so that + # it matches the behavior of the built-in class (it's odd though + # that the threading.RLock raises a runtime error on this same + # method instead...) + raise threading.ThreadError('Release attempted on unlocked lock') + # Cleans off one level of the stack (this is done so that if there + # are multiple __enter__() and __exit__() pairs active that this will + # only remove one level (the last one), and not all levels... + leftover = self._lock_stacks[-1] + while leftover: + lock = self._locks[leftover - 1] try: - if locked: - self._locks[i].release() - self._locked[i] = False - except threading.ThreadError: - LOG.exception("Unable to release lock %s", i + 1) + lock.release() + except (threading.ThreadError, RuntimeError) as e: + # Ensure that we adjust the lock stack under failure so that + # if release is attempted again that we do not try to release + # the locks we already released... + self._lock_stacks[-1] = leftover + raise threading.ThreadError( + "Unable to release lock %s/%s due to '%s'" + % (leftover, len(self._locks), e)) + else: + leftover -= 1 + # At the end only clear it off, so that under partial failure we don't + # lose any locks... + self._lock_stacks.pop() class _InterProcessLock(object): From dbf117aaf5eaf59a8b5d3c0516add2b708ce3e73 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 22 Sep 2014 21:20:15 -0700 Subject: [PATCH 040/240] Documentation cleanups and tweaks Apply some adjustments on the docs by rewording certain statements, linking to the correct classes and methods and linking to the correct exceptions to make it easier for users to follow the docs and there associated types and links. Change-Id: I03aac77c814fc4c376003f09a45559a0995b3c6c --- doc/source/arguments_and_results.rst | 178 +++++++++++++----------- doc/source/atoms.rst | 2 +- doc/source/engines.rst | 58 ++++---- doc/source/inputs_and_outputs.rst | 27 ++-- doc/source/notifications.rst | 27 ++-- doc/source/types.rst | 10 ++ taskflow/engines/worker_based/worker.py | 38 ++--- taskflow/utils/misc.py | 31 +++-- 8 files changed, 199 insertions(+), 172 deletions(-) diff --git a/doc/source/arguments_and_results.rst b/doc/source/arguments_and_results.rst index e23a6375..12ad6edd 100644 --- a/doc/source/arguments_and_results.rst +++ b/doc/source/arguments_and_results.rst @@ -1,32 +1,35 @@ -========================== -Atom Arguments and Results -========================== +===================== +Arguments and results +===================== .. |task.execute| replace:: :py:meth:`~taskflow.task.BaseTask.execute` .. |task.revert| replace:: :py:meth:`~taskflow.task.BaseTask.revert` .. |retry.execute| replace:: :py:meth:`~taskflow.retry.Retry.execute` .. |retry.revert| replace:: :py:meth:`~taskflow.retry.Retry.revert` +.. |Retry| replace:: :py:class:`~taskflow.retry.Retry` +.. |Task| replace:: :py:class:`Task ` -In TaskFlow, all flow and task state goes to (potentially persistent) storage. -That includes all the information that :doc:`atoms ` (e.g. tasks) in the -flow need when they are executed, and all the information task produces (via -serializable task results). A developer who implements tasks or flows can -specify what arguments a task accepts and what result it returns in several -ways. This document will help you understand what those ways are and how to use -those ways to accomplish your desired usage pattern. +In TaskFlow, all flow and task state goes to (potentially persistent) storage +(see :doc:`persistence ` for more details). That includes all the +information that :doc:`atoms ` (e.g. tasks, retry objects...) in the +workflow need when they are executed, and all the information task/retry +produces (via serializable results). A developer who implements tasks/retries +or flows can specify what arguments a task/retry accepts and what result it +returns in several ways. This document will help you understand what those ways +are and how to use those ways to accomplish your desired usage pattern. .. glossary:: - Task arguments - Set of names of task arguments available as the ``requires`` - property of the task instance. When a task is about to be executed - values with these names are retrieved from storage and passed to - |task.execute| method of the task. + Task/retry arguments + Set of names of task/retry arguments available as the ``requires`` + property of the task/retry instance. When a task or retry object is + about to be executed values with these names are retrieved from storage + and passed to the ``execute`` method of the task/retry. - Task results - Set of names of task results (what task provides) available as - ``provides`` property of task instance. After a task finishes - successfully, its result(s) (what the task |task.execute| method + Task/retry results + Set of names of task/retry results (what task/retry provides) available + as ``provides`` property of task or retry instance. After a task/retry + finishes successfully, its result(s) (what the ``execute`` method returns) are available by these names from storage (see examples below). @@ -44,8 +47,8 @@ There are different ways to specify the task argument ``requires`` set. Arguments inference ------------------- -Task arguments can be inferred from arguments of the |task.execute| method of -the task. +Task/retry arguments can be inferred from arguments of the |task.execute| +method of a task (or the |retry.execute| of a retry object). .. doctest:: @@ -56,10 +59,10 @@ the task. >>> sorted(MyTask().requires) ['eggs', 'spam'] -Inference from the method signature is the ''simplest'' way to specify task +Inference from the method signature is the ''simplest'' way to specify arguments. Optional arguments (with default values), and special arguments like -``self``, ``*args`` and ``**kwargs`` are ignored on inference (as these names -have special meaning/usage in python). +``self``, ``*args`` and ``**kwargs`` are ignored during inference (as these +names have special meaning/usage in python). .. doctest:: @@ -83,14 +86,14 @@ have special meaning/usage in python). Rebinding --------- -**Why:** There are cases when the value you want to pass to a task is stored -with a name other then the corresponding task arguments name. That's when the -``rebind`` task constructor parameter comes in handy. Using it the flow author +**Why:** There are cases when the value you want to pass to a task/retry is +stored with a name other then the corresponding arguments name. That's when the +``rebind`` constructor parameter comes in handy. Using it the flow author can instruct the engine to fetch a value from storage by one name, but pass it -to a tasks |task.execute| method with another name. There are two possible ways -of accomplishing this. +to a tasks/retrys ``execute`` method with another name. There are two possible +ways of accomplishing this. -The first is to pass a dictionary that maps the task argument name to the name +The first is to pass a dictionary that maps the argument name to the name of a saved value. For example, if you have task:: @@ -100,24 +103,25 @@ For example, if you have task:: def execute(self, vm_name, vm_image_id, **kwargs): pass # TODO(imelnikov): use parameters to spawn vm -and you saved 'vm_name' with 'name' key in storage, you can spawn a vm with -such 'name' like this:: +and you saved ``'vm_name'`` with ``'name'`` key in storage, you can spawn a vm +with such ``'name'`` like this:: SpawnVMTask(rebind={'vm_name': 'name'}) The second way is to pass a tuple/list/dict of argument names. The length of -the tuple/list/dict should not be less then number of task required parameters. +the tuple/list/dict should not be less then number of required parameters. + For example, you can achieve the same effect as the previous example with:: SpawnVMTask(rebind_args=('name', 'vm_image_id')) -which is equivalent to a more elaborate:: +This is equivalent to a more elaborate:: SpawnVMTask(rebind=dict(vm_name='name', vm_image_id='vm_image_id')) -In both cases, if your task accepts arbitrary arguments with ``**kwargs`` -construct, you can specify extra arguments. +In both cases, if your task (or retry) accepts arbitrary arguments +with the ``**kwargs`` construct, you can specify extra arguments. :: @@ -158,7 +162,8 @@ arguments) will appear in the ``kwargs`` of the |task.execute| method. When constructing a task instance the flow author can also add more requirements if desired. Those manual requirements (if they are not functional -arguments) will appear in the ``**kwargs`` the |task.execute| method. +arguments) will appear in the ``kwargs`` parameter of the |task.execute| +method. .. doctest:: @@ -189,12 +194,13 @@ avoid invalid argument mappings. Results specification ===================== -In python, function results are not named, so we can not infer what a task -returns. This is important since the complete task result (what the -|task.execute| method returns) is saved in (potentially persistent) storage, -and it is typically (but not always) desirable to make those results accessible -to other tasks. To accomplish this the task specifies names of those values via -its ``provides`` task constructor parameter or other method (see below). +In python, function results are not named, so we can not infer what a +task/retry returns. This is important since the complete result (what the +task |task.execute| or retry |retry.execute| method returns) is saved +in (potentially persistent) storage, and it is typically (but not always) +desirable to make those results accessible to others. To accomplish this +the task/retry specifies names of those values via its ``provides`` constructor +parameter or by its default provides attribute. Returning one value ------------------- @@ -242,14 +248,14 @@ tasks) will be able to get those elements from storage by name: Provides argument can be shorter then the actual tuple returned by a task -- then extra values are ignored (but, as expected, **all** those values are saved -and passed to the |task.revert| method). +and passed to the task |task.revert| or retry |retry.revert| method). .. note:: Provides arguments tuple can also be longer then the actual tuple returned by task -- when this happens the extra parameters are left undefined: a warning is printed to logs and if use of such parameter is attempted a - ``NotFound`` exception is raised. + :py:class:`~taskflow.exceptions.NotFound` exception is raised. Returning a dictionary ---------------------- @@ -290,16 +296,17 @@ will be able to get elements from storage by name: and passed to the |task.revert| method). If the provides argument has some items not present in the actual dict returned by the task -- then extra parameters are left undefined: a warning is printed to logs and if use of - such parameter is attempted a ``NotFound`` exception is raised. + such parameter is attempted a :py:class:`~taskflow.exceptions.NotFound` + exception is raised. Default provides ---------------- -As mentioned above, the default task base class provides nothing, which means -task results are not accessible to other tasks in the flow. +As mentioned above, the default base class provides nothing, which means +results are not accessible to other tasks/retrys in the flow. -The task author can override this and specify default value for provides using -``default_provides`` class variable: +The author can override this and specify default value for provides using +the ``default_provides`` class/instance variable: :: @@ -314,8 +321,8 @@ Of course, the flow author can override this to change names if needed: BitsAndPiecesTask(provides=('b', 'p')) -or to change structure -- e.g. this instance will make whole tuple accessible -to other tasks by name 'bnp': +or to change structure -- e.g. this instance will make tuple accessible +to other tasks by name ``'bnp'``: :: @@ -331,26 +338,27 @@ the task from other tasks in the flow (e.g. to avoid naming conflicts): Revert arguments ================ -To revert a task engine calls its |task.revert| method. This method -should accept same arguments as |task.execute| method of the task and one -more special keyword argument, named ``result``. +To revert a task the :doc:`engine ` calls the tasks +|task.revert| method. This method should accept the same arguments +as the |task.execute| method of the task and one more special keyword +argument, named ``result``. For ``result`` value, two cases are possible: -* if task is being reverted because it failed (an exception was raised from its - |task.execute| method), ``result`` value is instance of - :py:class:`taskflow.utils.misc.Failure` object that holds exception - information; +* If the task is being reverted because it failed (an exception was raised + from its |task.execute| method), the ``result`` value is an instance of a + :py:class:`~taskflow.utils.misc.Failure` object that holds the exception + information. -* if task is being reverted because some other task failed, and this task - finished successfully, ``result`` value is task result fetched from storage: - basically, that's what |task.execute| method returned. +* If the task is being reverted because some other task failed, and this task + finished successfully, ``result`` value is the result fetched from storage: + ie, what the |task.execute| method returned. All other arguments are fetched from storage in the same way it is done for |task.execute| method. -To determine if task failed you can check whether ``result`` is instance of -:py:class:`taskflow.utils.misc.Failure`:: +To determine if a task failed you can check whether ``result`` is instance of +:py:class:`~taskflow.utils.misc.Failure`:: from taskflow.utils import misc @@ -366,20 +374,21 @@ To determine if task failed you can check whether ``result`` is instance of else: print("do_something returned %r" % result) -If this task failed (``do_something`` raised exception) it will print ``"This -task failed, exception:"`` and exception message on revert. If this task -finished successfully, it will print ``"do_something returned"`` and -representation of result. +If this task failed (ie ``do_something`` raised an exception) it will print +``"This task failed, exception:"`` and a exception message on revert. If this +task finished successfully, it will print ``"do_something returned"`` and a +representation of the ``do_something`` result. Retry arguments =============== -A Retry controller works with arguments in the same way as a Task. But it has -an additional parameter 'history' that is a list of tuples. Each tuple contains -a result of the previous Retry run and a table where a key is a failed task and -a value is a :py:class:`taskflow.utils.misc.Failure`. +A |Retry| controller works with arguments in the same way as a |Task|. But +it has an additional parameter ``'history'`` that is a list of tuples. Each +tuple contains a result of the previous retry run and a table where the key +is a failed task and the value is a +:py:class:`~taskflow.utils.misc.Failure` object. -Consider the following Retry:: +Consider the following implementation:: class MyRetry(retry.Retry): @@ -396,19 +405,24 @@ Consider the following Retry:: def revert(self, history, *args, **kwargs): print history -Imagine the following Retry had returned a value '5' and then some task 'A' -failed with some exception. In this case ``on_failure`` method will receive -the following history:: +Imagine the above retry had returned a value ``'5'`` and then some task ``'A'`` +failed with some exception. In this case the above retrys ``on_failure`` +method will receive the following history:: [('5', {'A': misc.Failure()})] -Then the |retry.execute| method will be called again and it'll receive the same -history. +At this point (since the implementation returned ``RETRY``) the +|retry.execute| method will be called again and it will receive the same +history and it can then return a value that subseqent tasks can use to alter +there behavior. -If the |retry.execute| method raises an exception, the |retry.revert| method of -Retry will be called and :py:class:`taskflow.utils.misc.Failure` object will be -present in the history instead of Retry result:: +If instead the |retry.execute| method raises an exception, the |retry.revert| +method of the implementation will be called and +a :py:class:`~taskflow.utils.misc.Failure` object will be present in the +history instead of the typical result:: [('5', {'A': misc.Failure()}), (misc.Failure(), {})] -After the Retry has been reverted, the Retry history will be cleaned. +.. note:: + + After a |Retry| has been reverted, the objects history will be cleaned. diff --git a/doc/source/atoms.rst b/doc/source/atoms.rst index 85086346..26ca6ad0 100644 --- a/doc/source/atoms.rst +++ b/doc/source/atoms.rst @@ -1,5 +1,5 @@ ------------------------ -Atoms, Tasks and Retries +Atoms, tasks and retries ------------------------ Atom diff --git a/doc/source/engines.rst b/doc/source/engines.rst index 752f9f0e..45113a37 100644 --- a/doc/source/engines.rst +++ b/doc/source/engines.rst @@ -13,23 +13,23 @@ and uses it to decide which :doc:`atom ` to run and when. TaskFlow provides different implementations of engines. Some may be easier to use (ie, require no additional infrastructure setup) and understand; others might require more complicated setup but provide better scalability. The idea -and *ideal* is that deployers or developers of a service that uses TaskFlow can +and *ideal* is that deployers or developers of a service that use TaskFlow can select an engine that suites their setup best without modifying the code of said service. Engines usually have different capabilities and configuration, but all of them **must** implement the same interface and preserve the semantics of patterns -(e.g. parts of :py:class:`linear flow ` -are run one after another, in order, even if engine is *capable* of running -tasks in parallel). +(e.g. parts of a :py:class:`.linear_flow.Flow` +are run one after another, in order, even if the selected engine is *capable* +of running tasks in parallel). Why they exist -------------- -An engine being the core component which actually makes your flows progress is -likely a new concept for many programmers so let's describe how it operates in -more depth and some of the reasoning behind why it exists. This will hopefully -make it more clear on there value add to the TaskFlow library user. +An engine being *the* core component which actually makes your flows progress +is likely a new concept for many programmers so let's describe how it operates +in more depth and some of the reasoning behind why it exists. This will +hopefully make it more clear on there value add to the TaskFlow library user. First though let us discuss something most are familiar already with; the difference between `declarative`_ and `imperative`_ programming models. The @@ -48,15 +48,15 @@ more of a *pure* function that executes, reverts and may require inputs and provide outputs). This is where engines get involved; they do the execution of the *what* defined via :doc:`atoms `, tasks, flows and the relationships defined there-in and execute these in a well-defined manner (and the engine is -responsible for *most* of the state manipulation instead). +responsible for any state manipulation instead). This mix of imperative and declarative (with a stronger emphasis on the -declarative model) allows for the following functionality to be possible: +declarative model) allows for the following functionality to become possible: * Enhancing reliability: Decoupling of state alterations from what should be accomplished allows for a *natural* way of resuming by allowing the engine to - track the current state and know at which point a flow is in and how to get - back into that state when resumption occurs. + track the current state and know at which point a workflow is in and how to + get back into that state when resumption occurs. * Enhancing scalability: When a engine is responsible for executing your desired work it becomes possible to alter the *how* in the future by creating new types of execution backends (for example the worker model which does not @@ -83,13 +83,14 @@ Of course these kind of features can come with some drawbacks: away from (and this is likely a mindset change for programmers used to the imperative model). We have worked to make this less of a concern by creating and encouraging the usage of :doc:`persistence `, to help make - it possible to have some level of provided state transfer mechanism. + it possible to have state and tranfer that state via a argument input and + output mechanism. * Depending on how much imperative code exists (and state inside that code) - there can be *significant* rework of that code and converting or refactoring - it to these new concepts. We have tried to help here by allowing you to have - tasks that internally use regular python code (and internally can be written - in an imperative style) as well as by providing examples and these developer - docs; helping this process be as seamless as possible. + there *may* be *significant* rework of that code and converting or + refactoring it to these new concepts. We have tried to help here by allowing + you to have tasks that internally use regular python code (and internally can + be written in an imperative style) as well as by providing + :doc:`examples ` that show how to use these concepts. * Another one of the downsides of decoupling the *what* from the *how* is that it may become harder to use traditional techniques to debug failures (especially if remote workers are involved). We try to help here by making it @@ -110,7 +111,7 @@ All engines are mere classes that implement the same interface, and of course it is possible to import them and create instances just like with any classes in Python. But the easier (and recommended) way for creating an engine is using the engine helper functions. All of these functions are imported into the -`taskflow.engines` module namespace, so the typical usage of these functions +``taskflow.engines`` module namespace, so the typical usage of these functions might look like:: from taskflow import engines @@ -130,8 +131,8 @@ Usage To select which engine to use and pass parameters to an engine you should use the ``engine_conf`` parameter any helper factory function accepts. It may be: -* a string, naming engine type; -* a dictionary, holding engine type with key ``'engine'`` and possibly +* A string, naming the engine type. +* A dictionary, naming engine type with key ``'engine'`` and possibly type-specific engine configuration parameters. Single-Threaded @@ -139,15 +140,20 @@ Single-Threaded **Engine type**: ``'serial'`` -Runs all tasks on the single thread -- the same thread `engine.run()` is called -on. This engine is used by default. +Runs all tasks on a single thread -- the same thread ``engine.run()`` is +called from. + +.. note:: + + This engine is used by default. .. tip:: If eventlet is used then this engine will not block other threads - from running as eventlet automatically creates a co-routine system (using - greenthreads and monkey patching). See `eventlet `_ - and `greenlet `_ for more details. + from running as eventlet automatically creates a implicit co-routine + system (using greenthreads and monkey patching). See + `eventlet `_ and + `greenlet `_ for more details. Parallel -------- diff --git a/doc/source/inputs_and_outputs.rst b/doc/source/inputs_and_outputs.rst index 34fb1bad..e820fefe 100644 --- a/doc/source/inputs_and_outputs.rst +++ b/doc/source/inputs_and_outputs.rst @@ -1,11 +1,11 @@ ================== -Inputs and Outputs +Inputs and outputs ================== In TaskFlow there are multiple ways to provide inputs for your tasks and flows and get information from them. This document describes one of them, that involves task arguments and results. There are also :doc:`notifications -`, which allow you to get notified when task or flow changed +`, which allow you to get notified when a task or flow changes state. You may also opt to use the :doc:`persistence ` layer itself directly. @@ -19,15 +19,16 @@ This is the standard and recommended way to pass data from one task to another. Of course not every task argument needs to be provided to some other task of a flow, and not every task result should be consumed by every task. -If some value is required by one or more tasks of a flow, but is not provided -by any task, it is considered to be flow input, and **must** be put into the -storage before the flow is run. A set of names required by a flow can be -retrieved via that flow's ``requires`` property. These names can be used to +If some value is required by one or more tasks of a flow, but it is not +provided by any task, it is considered to be flow input, and **must** be put +into the storage before the flow is run. A set of names required by a flow can +be retrieved via that flow's ``requires`` property. These names can be used to determine what names may be applicable for placing in storage ahead of time and which names are not applicable. All values provided by tasks of the flow are considered to be flow outputs; the -set of names of such values is available via ``provides`` property of the flow. +set of names of such values is available via the ``provides`` property of the +flow. .. testsetup:: @@ -59,8 +60,10 @@ As you can see, this flow does not require b, as it is provided by the fist task. .. note:: - There is no difference between processing of Task and Retry inputs - and outputs. + + There is no difference between processing of + :py:class:`Task ` and + :py:class:`~taskflow.retry.Retry` inputs and outputs. ------------------ Engine and storage @@ -146,8 +149,10 @@ Outputs As you can see from examples above, the run method returns all flow outputs in a ``dict``. This same data can be fetched via -:py:meth:`~taskflow.storage.Storage.fetch_all` method of the storage. You can -also get single results using :py:meth:`~taskflow.storage.Storage.fetch`. +:py:meth:`~taskflow.storage.Storage.fetch_all` method of the engines storage +object. You can also get single results using the +engines storage objects :py:meth:`~taskflow.storage.Storage.fetch` method. + For example: .. doctest:: diff --git a/doc/source/notifications.rst b/doc/source/notifications.rst index 3fe430de..755f7b13 100644 --- a/doc/source/notifications.rst +++ b/doc/source/notifications.rst @@ -1,5 +1,5 @@ =========================== -Notifications and Listeners +Notifications and listeners =========================== .. testsetup:: @@ -17,9 +17,9 @@ transitions, which is useful for monitoring, logging, metrics, debugging and plenty of other tasks. To receive these notifications you should register a callback with -an instance of the the :py:class:`notifier ` +an instance of the :py:class:`~taskflow.utils.misc.Notifier` class that is attached -to :py:class:`engine ` +to :py:class:`Engine ` attributes ``task_notifier`` and ``notifier``. TaskFlow also comes with a set of predefined :ref:`listeners `, and @@ -30,17 +30,14 @@ using raw callbacks. Receiving notifications with callbacks -------------------------------------- -To manage notifications instances of -:py:class:`~taskflow.utils.misc.Notifier` are used. - -.. autoclass:: taskflow.utils.misc.Notifier - Flow notifications ------------------ -To receive notification on flow state changes use -:py:class:`~taskflow.utils.misc.Notifier` available as -``notifier`` property of the engine. A basic example is: +To receive notification on flow state changes use the +:py:class:`~taskflow.utils.misc.Notifier` instance available as the +``notifier`` property of an engine. + +A basic example is: .. doctest:: @@ -71,9 +68,11 @@ To receive notification on flow state changes use Task notifications ------------------ -To receive notification on task state changes use -:py:class:`~taskflow.utils.misc.Notifier` available as -``task_notifier`` property of the engine. A basic example is: +To receive notification on task state changes use the +:py:class:`~taskflow.utils.misc.Notifier` instance available as the +``task_notifier`` property of an engine. + +A basic example is: .. doctest:: diff --git a/doc/source/types.rst b/doc/source/types.rst index a5afa675..1f573c8a 100644 --- a/doc/source/types.rst +++ b/doc/source/types.rst @@ -7,6 +7,11 @@ Cache .. automodule:: taskflow.types.cache +Failure +======= + +.. autoclass:: taskflow.utils.misc.Failure + FSM === @@ -17,6 +22,11 @@ Graph .. automodule:: taskflow.types.graph +Notifier +======== + +.. autoclass:: taskflow.utils.misc.Notifier + Table ===== diff --git a/taskflow/engines/worker_based/worker.py b/taskflow/engines/worker_based/worker.py index 49816eab..ee3ea159 100644 --- a/taskflow/engines/worker_based/worker.py +++ b/taskflow/engines/worker_based/worker.py @@ -69,34 +69,16 @@ class Worker(object): :param url: broker url :param exchange: broker exchange name :param topic: topic name under which worker is stated - :param tasks: tasks list that worker is capable to perform - - Tasks list item can be one of the following types: - 1. String: - - 1.1 Python module name: - - > tasks=['taskflow.tests.utils'] - - 1.2. Task class (BaseTask subclass) name: - - > tasks=['taskflow.test.utils.DummyTask'] - - 3. Python module: - - > from taskflow.tests import utils - > tasks=[utils] - - 4. Task class (BaseTask subclass): - - > from taskflow.tests import utils - > tasks=[utils.DummyTask] - - :param executor: custom executor object that is used for processing - requests in separate threads - :keyword threads_count: threads count to be passed to the default executor - :keyword transport: transport to be used (e.g. amqp, memory, etc.) - :keyword transport_options: transport specific options + :param tasks: task list that worker is capable of performing, items in + the list can be one of the following types; 1, a string naming the + python module name to search for tasks in or the task class name; 2, a + python module to search for tasks in; 3, a task class object that + will be used to create tasks from. + :param executor: custom executor object that can used for processing + requests in separate threads (if not provided one will be created) + :param threads_count: threads count to be passed to the default executor + :param transport: transport to be used (e.g. amqp, memory, etc.) + :param transport_options: transport specific options """ def __init__(self, exchange, topic, tasks, executor=None, **kwargs): diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 035e86ac..39846216 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -416,7 +416,10 @@ class Notifier(object): notification occurs. """ + #: Keys that can not be used in callbacks arguments RESERVED_KEYS = ('details',) + + #: Kleene star constant that is used to recieve all notifications ANY = '*' def __init__(self): @@ -474,9 +477,9 @@ class Notifier(object): Callback will be called with provided ``args`` and ``kwargs`` and when event type occurs (or on any event if ``event_type`` equals to - ``Notifier.ANY``). It will also get additional keyword argument, - ``details``, that will hold event details provided to - :py:meth:`notify` method. + :attr:`.ANY`). It will also get additional keyword argument, + ``details``, that will hold event details provided to the + :meth:`.notify` method. """ assert six.callable(callback), "Callback must be callable" if self.is_registered(event_type, callback): @@ -576,9 +579,10 @@ class Failure(object): remote worker throws an exception, the WBE based engine will receive that exception and desire to reraise it to the user/caller of the WBE based engine for appropriate handling (this matches the behavior of non-remote - engines). To accomplish this a failure object (or a to_dict() form) would - be sent over the WBE channel and the WBE based engine would deserialize it - and use this objects reraise() method to cause an exception that contains + engines). To accomplish this a failure object (or a + :py:meth:`~misc.Failure.to_dict` form) would be sent over the WBE channel + and the WBE based engine would deserialize it and use this objects + :meth:`.reraise` method to cause an exception that contains similar/equivalent information as the original exception to be reraised, allowing the user (or the WBE engine itself) to then handle the worker failure/exception as they desire. @@ -642,6 +646,7 @@ class Failure(object): @classmethod def from_exception(cls, exception): + """Creates a failure object from a exception instance.""" return cls((type(exception), exception, None)) def _matches(self, other): @@ -652,6 +657,7 @@ class Failure(object): and self.traceback_str == other.traceback_str) def matches(self, other): + """Checks if another object is equivalent to this object.""" if not isinstance(other, Failure): return False if self.exc_info is None or other.exc_info is None: @@ -706,9 +712,10 @@ class Failure(object): """Re-raise exceptions if argument is not empty. If argument is empty list, this method returns None. If - argument is list with single Failure object in it, - this failure is reraised. Else, WrappedFailure exception - is raised with failures list as causes. + argument is a list with a single ``Failure`` object in it, + that failure is reraised. Else, a + :class:`~taskflow.exceptions.WrappedFailure` exception + is raised with a failure list as causes. """ failures = list(failures) if len(failures) == 1: @@ -724,7 +731,7 @@ class Failure(object): raise exc.WrappedFailure([self]) def check(self, *exc_classes): - """Check if any of exc_classes caused the failure. + """Check if any of ``exc_classes`` caused the failure. Arguments of this method can be exception types or type names (stings). If captured exception is instance of @@ -744,6 +751,7 @@ class Failure(object): return self.pformat() def pformat(self, traceback=False): + """Pretty formats the failure object into a string.""" buf = six.StringIO() buf.write( 'Failure: %s: %s' % (self._exc_type_names[0], self._exception_str)) @@ -766,6 +774,7 @@ class Failure(object): @classmethod def from_dict(cls, data): + """Converts this from a dictionary to a object.""" data = dict(data) version = data.pop('version', None) if version != cls.DICT_VERSION: @@ -774,6 +783,7 @@ class Failure(object): return cls(**data) def to_dict(self): + """Converts this object to a dictionary.""" return { 'exception_str': self.exception_str, 'traceback_str': self.traceback_str, @@ -782,6 +792,7 @@ class Failure(object): } def copy(self): + """Copies this object.""" return Failure(exc_info=copy_exc_info(self.exc_info), exception_str=self.exception_str, traceback_str=self.traceback_str, From b5e3fbebc6ea35f6e90936fbfed1c198e1045314 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 20 Sep 2014 10:40:35 -0700 Subject: [PATCH 041/240] Expand toctree to three levels Three levels makes it easier to find content in the main toctree so lets make it easier for folks to use the table of contents to find what they are looking for instead of making it harder... This change makes three levels look readable as well as fixes some discrepancies among the various sections... Change-Id: I5fd7a062adec052c338790c9ba343dfbc51075e3 --- doc/source/arguments_and_results.rst | 11 +++++++---- doc/source/atoms.rst | 4 ++-- doc/source/engines.rst | 9 ++++++--- doc/source/index.rst | 2 +- doc/source/jobs.rst | 11 +++++++++++ doc/source/persistence.rst | 3 +++ doc/source/resumption.rst | 14 +++++++------- 7 files changed, 37 insertions(+), 17 deletions(-) diff --git a/doc/source/arguments_and_results.rst b/doc/source/arguments_and_results.rst index 12ad6edd..d7b96095 100644 --- a/doc/source/arguments_and_results.rst +++ b/doc/source/arguments_and_results.rst @@ -202,8 +202,11 @@ desirable to make those results accessible to others. To accomplish this the task/retry specifies names of those values via its ``provides`` constructor parameter or by its default provides attribute. +Examples +-------- + Returning one value -------------------- ++++++++++++++++++++ If task returns just one value, ``provides`` should be string -- the name of the value. @@ -218,7 +221,7 @@ name of the value. set(['the_answer']) Returning a tuple ------------------ ++++++++++++++++++ For a task that returns several values, one option (as usual in python) is to return those values via a ``tuple``. @@ -258,7 +261,7 @@ and passed to the task |task.revert| or retry |retry.revert| method). :py:class:`~taskflow.exceptions.NotFound` exception is raised. Returning a dictionary ----------------------- +++++++++++++++++++++++ Another option is to return several values as a dictionary (aka a ``dict``). @@ -300,7 +303,7 @@ will be able to get elements from storage by name: exception is raised. Default provides ----------------- +++++++++++++++++ As mentioned above, the default base class provides nothing, which means results are not accessible to other tasks/retrys in the flow. diff --git a/doc/source/atoms.rst b/doc/source/atoms.rst index 26ca6ad0..f2b75ffa 100644 --- a/doc/source/atoms.rst +++ b/doc/source/atoms.rst @@ -94,8 +94,8 @@ subclasses are provided: :py:class:`~taskflow.retry.ForEach` but extracts values from storage instead of the :py:class:`~taskflow.retry.ForEach` constructor. -Usage ------ +Examples +-------- .. testsetup:: diff --git a/doc/source/engines.rst b/doc/source/engines.rst index 45113a37..be19aa45 100644 --- a/doc/source/engines.rst +++ b/doc/source/engines.rst @@ -135,8 +135,11 @@ the ``engine_conf`` parameter any helper factory function accepts. It may be: * A dictionary, naming engine type with key ``'engine'`` and possibly type-specific engine configuration parameters. -Single-Threaded ---------------- +Types +===== + +Serial +------ **Engine type**: ``'serial'`` @@ -180,7 +183,7 @@ Additional supported keyword arguments: Running tasks with a `process pool executor`_ is not currently supported. -Worker-Based +Worker-based ------------ **Engine type**: ``'worker-based'`` diff --git a/doc/source/index.rst b/doc/source/index.rst index 23ffcfcb..d3820470 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -14,7 +14,7 @@ Contents ======== .. toctree:: - :maxdepth: 2 + :maxdepth: 3 atoms arguments_and_results diff --git a/doc/source/jobs.rst b/doc/source/jobs.rst index 048a66ea..4c482b51 100644 --- a/doc/source/jobs.rst +++ b/doc/source/jobs.rst @@ -161,6 +161,9 @@ might look like: time.sleep(coffee_break_time) ... +Types +===== + Zookeeper --------- @@ -248,6 +251,14 @@ Interfaces .. automodule:: taskflow.jobs.job .. automodule:: taskflow.jobs.jobboard +Hierarchy +========= + +.. inheritance-diagram:: + taskflow.jobs.jobboard + taskflow.jobs.backends.impl_zookeeper + :parts: 1 + .. _paradigm shift: https://wiki.openstack.org/wiki/TaskFlow/Paradigm_shifts#Workflow_ownership_transfer .. _zookeeper: http://zookeeper.apache.org/ .. _kazoo: http://kazoo.readthedocs.org/ diff --git a/doc/source/persistence.rst b/doc/source/persistence.rst index 022773e5..18fe6bff 100644 --- a/doc/source/persistence.rst +++ b/doc/source/persistence.rst @@ -144,6 +144,9 @@ the following: ``'connection'`` and possibly type-specific backend parameters as other keys. +Types +===== + Memory ------ diff --git a/doc/source/resumption.rst b/doc/source/resumption.rst index 8ddd4e95..3be864f6 100644 --- a/doc/source/resumption.rst +++ b/doc/source/resumption.rst @@ -88,7 +88,7 @@ The following scenarios explain some expected structural changes and how they can be accommodated (and what the effect will be when resuming & running). Same atoms ----------- +++++++++++ When the factory function mentioned above returns the exact same the flow and atoms (no changes are performed). @@ -98,7 +98,7 @@ atoms with :py:class:`~taskflow.persistence.logbook.AtomDetail` objects by name and then the engine resumes. Atom was added --------------- +++++++++++++++ When the factory function mentioned above alters the flow by adding a new atom in (for example for changing the runtime structure of what was previously ran @@ -109,7 +109,7 @@ corresponding :py:class:`~taskflow.persistence.logbook.AtomDetail` does not exist and one will be created and associated. Atom was removed ----------------- +++++++++++++++++ When the factory function mentioned above alters the flow by removing a new atom in (for example for changing the runtime structure of what was previously @@ -121,7 +121,7 @@ it was not there, and any results it returned if it was completed before will be ignored. Atom code was changed ---------------------- ++++++++++++++++++++++ When the factory function mentioned above alters the flow by deciding that a newer version of a previously existing atom should be ran (possibly to perform @@ -137,8 +137,8 @@ ability to upgrade atoms before running (manual introspection & modification of a :py:class:`~taskflow.persistence.logbook.LogBook` can be done before engine loading and running to accomplish this in the meantime). -Atom was split in two atoms or merged from two (or more) to one atom --------------------------------------------------------------------- +Atom was split in two atoms or merged ++++++++++++++++++++++++++++++++++++++ When the factory function mentioned above alters the flow by deciding that a previously existing atom should be split into N atoms or the factory function @@ -154,7 +154,7 @@ introspection & modification of a loading and running to accomplish this in the meantime). Flow structure was changed --------------------------- +++++++++++++++++++++++++++ If manual links were added or removed from graph, or task requirements were changed, or flow was refactored (atom moved into or out of subflows, linear From 5ff4c22a8be842b8db2c1c59d0ac6bcd8a21dded Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Thu, 25 Sep 2014 10:08:44 +0200 Subject: [PATCH 042/240] Stop using intersphinx Remove intersphinx from the docs build as it triggers network calls that occasionally fail, and we don't really use intersphinx (links other sphinx documents out on the internet) This also removes the requirement for internet access during docs build. This can cause docs jobs to fail if the project errors out on warnings. Change-Id: Ie70e400478a092943a18f9f8595302380ec71cbc Related-Bug: #1368910 --- doc/source/conf.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 3b0c35ce..fcb07166 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -13,7 +13,6 @@ extensions = [ 'sphinx.ext.doctest', 'sphinx.ext.extlinks', 'sphinx.ext.inheritance_diagram', - 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode', 'oslosphinx' ] @@ -82,9 +81,6 @@ latex_documents = [ 'OpenStack Foundation', 'manual'), ] -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} - # -- Options for autoddoc ---------------------------------------------------- # Keep source order From 9537f523512eed8603dc304d8502762caaa733cf Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 25 Sep 2014 14:55:28 -0700 Subject: [PATCH 043/240] Document more function/class/method params To help fill out the docs start adding on docstrings that document what a parameter is and what its meaning and usage is to help users better understand the parameter. Part of ongoing bug 1374202 Change-Id: I9f9a83cfb763a0a05d22efca4e0f80627ba8ca8f --- taskflow/atom.py | 20 +++++++---- taskflow/exceptions.py | 39 ++++++++++++++++++---- taskflow/retry.py | 76 ++++++++++++++++++++++-------------------- taskflow/task.py | 30 ++++++++++------- 4 files changed, 103 insertions(+), 62 deletions(-) diff --git a/taskflow/atom.py b/taskflow/atom.py index d93ff57a..b3b7ea9c 100644 --- a/taskflow/atom.py +++ b/taskflow/atom.py @@ -125,7 +125,7 @@ class Atom(object): with this atom. It can be useful in resuming older versions of atoms. Standard major, minor versioning concepts should apply. - :ivar save_as: An *immutable* output ``resource`` name dict this atom + :ivar save_as: An *immutable* output ``resource`` name dictionary this atom produces that other atoms may depend on this atom providing. The format is output index (or key when a dictionary is returned from the execute method) to stored argument @@ -136,11 +136,19 @@ class Atom(object): the names that this atom expects (in a way this is like remapping a namespace of another atom into the namespace of this atom). - :ivar inject: An *immutable* input_name => value dictionary which specifies - any initial inputs that should be automatically injected into - the atoms scope before the atom execution commences (this - allows for providing atom *local* values that do not need to - be provided by other atoms). + :param name: Meaningful name for this atom, should be something that is + distinguishable and understandable for notification, + debugging, storing and any other similar purposes. + :param provides: A set, string or list of items that + this will be providing (or could provide) to others, used + to correlate and associate the thing/s this atom + produces, if it produces anything at all. + :param inject: An *immutable* input_name => value dictionary which + specifies any initial inputs that should be automatically + injected into the atoms scope before the atom execution + commences (this allows for providing atom *local* values that + do not need to be provided by other atoms/dependents). + :ivar inject: See parameter ``inject``. """ def __init__(self, name=None, provides=None, inject=None): diff --git a/taskflow/exceptions.py b/taskflow/exceptions.py index 21bf35ee..1bdb0f48 100644 --- a/taskflow/exceptions.py +++ b/taskflow/exceptions.py @@ -25,6 +25,14 @@ class TaskFlowException(Exception): NOTE(harlowja): in later versions of python we can likely remove the need to have a cause here as PY3+ have implemented PEP 3134 which handles chaining in a much more elegant manner. + + :param message: the exception message, typically some string that is + useful for consumers to view when debugging or analyzing + failures. + :param cause: the cause of the exception being raised, when provided this + should itself be an exception instance, this is useful for + creating a chain of exceptions for versions of python where + this is not yet implemented/supported natively. """ def __init__(self, message, cause=None): super(TaskFlowException, self).__init__(message) @@ -99,7 +107,16 @@ class DependencyFailure(TaskFlowException): class MissingDependencies(DependencyFailure): - """Raised when a entity has dependencies that can not be satisfied.""" + """Raised when a entity has dependencies that can not be satisfied. + + :param who: the entity that caused the missing dependency to be triggered. + :param requirements: the dependency which were not satisfied. + + Further arguments are interpreted as for in + :py:class:`~taskflow.exceptions.TaskFlowException`. + """ + + #: Exception message template used when creating an actual message. MESSAGE_TPL = ("%(who)s requires %(requirements)s but no other entity" " produces said requirements") @@ -147,6 +164,9 @@ class WrappedFailure(Exception): See the failure class documentation for a more comprehensive set of reasons why this object *may* be reraised instead of the original exception. + + :param causes: the :py:class:`~taskflow.utils.misc.Failure` objects that + caused this this exception to be raised. """ def __init__(self, causes): @@ -168,12 +188,14 @@ class WrappedFailure(Exception): return len(self._causes) def check(self, *exc_classes): - """Check if any of exc_classes caused (part of) the failure. + """Check if any of exception classes caused the failure/s. - Arguments of this method can be exception types or type names - (strings). If any of wrapped failures were caused by exception - of given type, the corresponding argument is returned. Else, - None is returned. + :param exc_classes: exception types/exception type names to + search for. + + If any of the contained failures were caused by an exception of a + given type, the corresponding argument that matched is returned. If + not then none is returned. """ if not exc_classes: return None @@ -189,7 +211,10 @@ class WrappedFailure(Exception): def exception_message(exc): - """Return the string representation of exception.""" + """Return the string representation of exception. + + :param exc: exception object to get a string representation of. + """ # NOTE(imelnikov): Dealing with non-ascii data in python is difficult: # https://bugs.launchpad.net/taskflow/+bug/1275895 # https://bugs.launchpad.net/taskflow/+bug/1276053 diff --git a/taskflow/retry.py b/taskflow/retry.py index 425c8ea6..edfc4d18 100644 --- a/taskflow/retry.py +++ b/taskflow/retry.py @@ -33,43 +33,19 @@ RETRY = "RETRY" @six.add_metaclass(abc.ABCMeta) -class Decider(object): - """A class/mixin object that can decide how to resolve execution failures. - - A decider may be executed multiple times on subflow or other atom - failure and it is expected to make a decision about what should be done - to resolve the failure (retry, revert to the previous retry, revert - the whole flow, etc.). - """ - - @abc.abstractmethod - def on_failure(self, history, *args, **kwargs): - """On failure makes a decision about the future. - - This method will typically use information about prior failures (if - this historical failure information is not available or was not - persisted this history will be empty). - - Returns retry action constant: - - * ``RETRY`` when subflow must be reverted and restarted again (maybe - with new parameters). - * ``REVERT`` when this subflow must be completely reverted and parent - subflow should make a decision about the flow execution. - * ``REVERT_ALL`` in a case when the whole flow must be reverted and - marked as ``FAILURE``. - """ - - -@six.add_metaclass(abc.ABCMeta) -class Retry(atom.Atom, Decider): +class Retry(atom.Atom): """A class that can decide how to resolve execution failures. This abstract base class is used to inherit from and provide different strategies that will be activated upon execution failures. Since a retry - object is an atom it may also provide execute and revert methods to alter - the inputs of connected atoms (depending on the desired strategy to be - used this can be quite useful). + object is an atom it may also provide :meth:`.execute` and + :meth:`.revert` methods to alter the inputs of connected atoms (depending + on the desired strategy to be used this can be quite useful). + + NOTE(harlowja): the :meth:`.execute` and :meth:`.revert` and + :meth:`.on_failure` will automatically be given a ``history`` parameter, + which contains information about the past decisions and outcomes + that have occurred (if available). """ default_provides = None @@ -92,11 +68,11 @@ class Retry(atom.Atom, Decider): @abc.abstractmethod def execute(self, history, *args, **kwargs): - """Executes the given retry atom. + """Executes the given retry. This execution activates a given retry which will typically produce data required to start or restart a connected component using - previously provided values and a history of prior failures from + previously provided values and a ``history`` of prior failures from previous runs. The historical data can be analyzed to alter the resolution strategy that this retry controller will use. @@ -105,12 +81,15 @@ class Retry(atom.Atom, Decider): saved to the history of the retry atom automatically, that is a list of tuples (result, failures) are persisted where failures is a dictionary of failures indexed by task names and the result is the execution - result returned by this retry controller during that failure resolution + result returned by this retry during that failure resolution attempt. + + :param args: positional arguments that retry requires to execute. + :param kwargs: any keyword arguments that retry requires to execute. """ def revert(self, history, *args, **kwargs): - """Reverts this retry using the given context. + """Reverts this retry. On revert call all results that had been provided by previous tries and all errors caused during reversion are provided. This method @@ -118,6 +97,29 @@ class Retry(atom.Atom, Decider): retry (that is to say that the controller has ran out of resolution options and has either given up resolution or has failed to handle a execution failure). + + :param args: positional arguments that the retry required to execute. + :param kwargs: any keyword arguments that the retry required to + execute. + """ + + @abc.abstractmethod + def on_failure(self, history, *args, **kwargs): + """Makes a decision about the future. + + This method will typically use information about prior failures (if + this historical failure information is not available or was not + persisted the provided history will be empty). + + Returns a retry constant (one of): + + * ``RETRY``: when the controlling flow must be reverted and restarted + again (for example with new parameters). + * ``REVERT``: when this controlling flow must be completely reverted + and the parent flow (if any) should make a decision about further + flow execution. + * ``REVERT_ALL``: when this controlling flow and the parent + flow (if any) must be reverted and marked as a ``FAILURE``. """ diff --git a/taskflow/task.py b/taskflow/task.py index cd470e72..62fd7314 100644 --- a/taskflow/task.py +++ b/taskflow/task.py @@ -52,7 +52,7 @@ class BaseTask(atom.Atom): A common pattern for initializing the state of the system prior to running tasks is to define some code in a base class that all your - tasks inherit from. In that class, you can define a pre_execute + tasks inherit from. In that class, you can define a ``pre_execute`` method and it will always be invoked just prior to your tasks running. """ @@ -72,6 +72,9 @@ class BaseTask(atom.Atom): happens in a different python process or on a remote machine) and so that the result can be transmitted to other tasks (which may be local or remote). + + :param args: positional arguments that task requires to execute. + :param kwargs: any keyword arguments that task requires to execute. """ def post_execute(self): @@ -79,7 +82,7 @@ class BaseTask(atom.Atom): A common pattern for cleaning up global state of the system after the execution of tasks is to define some code in a base class that all your - tasks inherit from. In that class, you can define a post_execute + tasks inherit from. In that class, you can define a ``post_execute`` method and it will always be invoked just after your tasks execute, regardless of whether they succeded or not. @@ -90,7 +93,7 @@ class BaseTask(atom.Atom): def pre_revert(self): """Code to be run prior to reverting the task. - This works the same as pre_execute, but for the revert phase. + This works the same as :meth:`.pre_execute`, but for the revert phase. """ def revert(self, *args, **kwargs): @@ -98,26 +101,29 @@ class BaseTask(atom.Atom): This method should undo any side-effects caused by previous execution of the task using the result of the :py:meth:`execute` method and - information on failure which triggered reversion of the flow. + information on the failure which triggered reversion of the flow the + task is contained in (if applicable). - NOTE(harlowja): The ``**kwargs`` which are passed into the - :py:meth:`execute` method will also be passed into this method. The - ``**kwargs`` key ``'result'`` will contain the :py:meth:`execute` - result (if any) and the ``**kwargs`` key ``'flow_failures'`` will - contain the failure information. + :param args: positional arguments that the task required to execute. + :param kwargs: any keyword arguments that the task required to + execute; the special key ``'result'`` will contain + the :py:meth:`execute` result (if any) and + the ``**kwargs`` key ``'flow_failures'`` will contain + any failure information. """ def post_revert(self): """Code to be run after reverting the task. - This works the same as post_execute, but for the revert phase. + This works the same as :meth:`.post_execute`, but for the revert phase. """ def update_progress(self, progress, **kwargs): """Update task progress and notify all registered listeners. - :param progress: task progress float value between 0 and 1 - :param kwargs: task specific progress information + :param progress: task progress float value between 0.0 and 1.0 + :param kwargs: any keyword arguments that are tied to the specific + progress value. """ if progress > 1.0: LOG.warn("Progress must be <= 1.0, clamping to upper bound") From 1e216c61f11ff7efe9cf5e4b205c8f418cf76fc0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 9 Sep 2014 10:40:38 -0700 Subject: [PATCH 044/240] Move some of the custom requirements out of tox.ini This change removes the need for an optional-requirements file which isn't being kept up to date with the rest of the openstack ecosystem and moves most of the customizations to test-requirements and where different versions are still needed (aka for SQLAlchemy) we place those varations into there needed tox environment as required. Change-Id: I443794b83de3f6a196fa7fc29a90620fb51b7f4c --- optional-requirements.txt | 31 ------------------------------- requirements-py2.txt | 6 ++++++ requirements-py3.txt | 4 ++++ test-requirements.txt | 17 ++++++++++++++++- tox.ini | 22 ++++++++++------------ 5 files changed, 36 insertions(+), 44 deletions(-) delete mode 100644 optional-requirements.txt diff --git a/optional-requirements.txt b/optional-requirements.txt deleted file mode 100644 index e010cf60..00000000 --- a/optional-requirements.txt +++ /dev/null @@ -1,31 +0,0 @@ -# This file lists dependencies that are used by different pluggable (optional) -# parts of TaskFlow, like engines or persistence backends. They are not -# strictly required by TaskFlow (aka you can use TaskFlow without them), so -# they don't go into one of the requirements.txt files. - -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. - -# Database (sqlalchemy) persistence: -SQLAlchemy>=0.7.8,<=0.9.99 -alembic>=0.4.1 - -# Database (sqlalchemy) persistence with MySQL: -MySQL-python - -# NOTE(imelnikov): pyMySQL should be here, but for now it's commented out -# because of https://bugs.launchpad.net/openstack-ci/+bug/1280008 -# pyMySQL - -# Database (sqlalchemy) persistence with PostgreSQL: -psycopg2 - -# ZooKeeper backends -kazoo>=1.3.1 - -# Eventlet may be used with parallel engine: -eventlet>=0.13.0 - -# Needed for the worker-based engine: -kombu>=2.4.8 diff --git a/requirements-py2.txt b/requirements-py2.txt index d4f85ae7..3f6292dd 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -6,16 +6,22 @@ # Only needed on python 2.6 ordereddict + # Python 2->3 compatibility library. six>=1.7.0 + # Very nice graph library networkx>=1.8 + # Used for backend storage engine loading. stevedore>=1.0.0 # Apache-2.0 + # Backport for concurrent.futures which exists in 3.2+ futures>=2.1.6 + # Used for structured input validation jsonschema>=2.0.0,<3.0.0 + # For common utilities oslo.utils>=0.3.0 oslo.serialization>=0.1.0 diff --git a/requirements-py3.txt b/requirements-py3.txt index 59ee1d36..8befdafc 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -6,12 +6,16 @@ # Python 2->3 compatibility library. six>=1.7.0 + # Very nice graph library networkx>=1.8 + # Used for backend storage engine loading. stevedore>=1.0.0 # Apache-2.0 + # Used for structured input validation jsonschema>=2.0.0,<3.0.0 + # For common utilities oslo.utils>=0.3.0 oslo.serialization>=0.1.0 diff --git a/test-requirements.txt b/test-requirements.txt index bbfe80b2..b72ca0b8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -9,7 +9,22 @@ mock>=1.0 python-subunit>=0.0.18 testrepository>=0.0.18 testtools>=0.9.34 + +# Used for testing the WBE engine. +kombu>=2.4.8 + +# Used for testing zookeeper & backends. zake>=0.1 # Apache-2.0 -# docs build jobs +kazoo>=1.3.1 + +# Used for testing database persistence backends. +# +# NOTE(harlowja): SQLAlchemy isn't listed here currently but is +# listed in our tox.ini files so that we can test multiple varying SQLAlchemy +# versions to ensure a wider range of compatibility. +alembic>=0.6.4 +psycopg2 + +# Docs build jobs need these packages. sphinx>=1.1.2,!=1.2.0,<1.3 oslosphinx>=2.2.0 # Apache-2.0 diff --git a/tox.ini b/tox.ini index 83030ce6..bc912778 100644 --- a/tox.ini +++ b/tox.ini @@ -17,10 +17,6 @@ usedevelop = True install_command = pip install {opts} {packages} setenv = VIRTUAL_ENV={envdir} deps = -r{toxinidir}/test-requirements.txt - alembic>=0.4.1 - psycopg2 - kazoo>=1.3.1 - kombu>=2.4.8 commands = python setup.py testr --slowest --testr-args='{posargs}' [testenv:docs] @@ -61,18 +57,20 @@ exclude = .venv,.tox,dist,doc,./taskflow/openstack/common,*egg,.git,build,tools # NOTE(imelnikov): pyXY envs are considered to be default, so they must have # richest set of test requirements [testenv:py26] +basepython = python2.6 deps = {[testenv]deps} -r{toxinidir}/requirements-py2.txt - SQLAlchemy>=0.7.8,<=0.7.99 MySQL-python eventlet>=0.13.0 -basepython = python2.6 + SQLAlchemy>=0.7.8,<=0.7.99 [testenv:py27] -deps = -r{toxinidir}/requirements-py2.txt - -r{toxinidir}/optional-requirements.txt - -r{toxinidir}/test-requirements.txt - doc8>=0.3.4 +deps = {[testenv]deps} + -r{toxinidir}/requirements-py2.txt + MySQL-python + eventlet>=0.13.0 + SQLAlchemy>=0.7.8,<=0.9.99 + doc8 commands = python setup.py testr --slowest --testr-args='{posargs}' sphinx-build -b doctest doc/source doc/build @@ -89,15 +87,15 @@ deps = {[testenv]deps} SQLAlchemy>=0.7.8,<=0.9.99 [testenv:py26-sa7-mysql] +basepython = python2.6 deps = {[testenv]deps} -r{toxinidir}/requirements-py2.txt SQLAlchemy>=0.7.8,<=0.7.99 MySQL-python -basepython = python2.6 [testenv:py27-sa8-mysql] +basepython = python2.7 deps = {[testenv]deps} -r{toxinidir}/requirements-py2.txt SQLAlchemy>=0.8,<=0.8.99 MySQL-python -basepython = python2.7 From 7640b09250da85271bf69a0e6553be699b195066 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 3 Sep 2014 18:17:14 -0700 Subject: [PATCH 045/240] Bring in a newer optional eventlet Since the openstack requirements repo just accepted eventlet>=0.15.1 we might as well also use that version in our optional requirements and remove a piece of code that was dealing with a bug that was fixed in eventlet 0.15. Change-Id: I9b4f9061c7adb7d72315315f41bb0d742b6f56b5 --- taskflow/utils/eventlet_utils.py | 4 +--- tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/taskflow/utils/eventlet_utils.py b/taskflow/utils/eventlet_utils.py index cc26dfe1..335dd017 100644 --- a/taskflow/utils/eventlet_utils.py +++ b/taskflow/utils/eventlet_utils.py @@ -144,9 +144,7 @@ class GreenExecutor(futures.Executor): self._shutdown = True if wait: self._pool.waitall() - # NOTE(harlowja): Fixed in eventlet 0.15 (remove when able to use) - if not self._delayed_work.empty(): - self._delayed_work.join() + self._delayed_work.join() class _GreenWaiter(object): diff --git a/tox.ini b/tox.ini index bc912778..186b438f 100644 --- a/tox.ini +++ b/tox.ini @@ -61,14 +61,14 @@ basepython = python2.6 deps = {[testenv]deps} -r{toxinidir}/requirements-py2.txt MySQL-python - eventlet>=0.13.0 + eventlet>=0.15.1 SQLAlchemy>=0.7.8,<=0.7.99 [testenv:py27] deps = {[testenv]deps} -r{toxinidir}/requirements-py2.txt MySQL-python - eventlet>=0.13.0 + eventlet>=0.15.1 SQLAlchemy>=0.7.8,<=0.9.99 doc8 commands = From f8d69ffc314cbab4c28a70db3c4f772c6af60fcb Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 16 Jul 2014 19:53:45 -0700 Subject: [PATCH 046/240] Adjust on_job_posting to not hold the lock while investigating To avoid issues when investigating jobs in one thread and having a dispatcher thread get locked on that same investigation lock avoid this entirely by not holding onto that lock. This also adds a small cleanup around the notifier thread pool and ensures that it actually exists in a useable state (and handles the exceptions that arise when it doesn't) on submission. Change-Id: I12e71f029726ec54ec0aadbdf71cc3c7a959f047 --- taskflow/jobs/backends/impl_zookeeper.py | 54 +++++++++++++++--------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/taskflow/jobs/backends/impl_zookeeper.py b/taskflow/jobs/backends/impl_zookeeper.py index cc6101c1..96307790 100644 --- a/taskflow/jobs/backends/impl_zookeeper.py +++ b/taskflow/jobs/backends/impl_zookeeper.py @@ -312,8 +312,12 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): def _emit(self, state, details): # Submit the work to the executor to avoid blocking the kazoo queue. - if self._worker is not None: + try: self._worker.submit(self.notifier.notify, state, details) + except (AttributeError, RuntimeError): + # Notification thread is shutdown or non-existent, either case we + # just want to skip submitting a notification... + pass @property def path(self): @@ -382,6 +386,9 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): else: self._job_cond.acquire() try: + # Now we can offically check if someone already placed this + # jobs information into the known job set (if it's already + # existing then just leave it alone). if path not in self._known_jobs: job = ZookeeperJob(job_data['name'], self, self._client, self._persistence, path, @@ -405,32 +412,37 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): continue child_paths.append(k_paths.join(self.path, c)) - # Remove jobs that we know about but which are no longer children + # Figure out what we really should be investigating and what we + # shouldn't... + investigate_paths = [] with self._job_lock: removals = set() - for path, _job in six.iteritems(self._known_jobs): + for path in six.iterkeys(self._known_jobs): if path not in child_paths: removals.add(path) for path in removals: self._remove_job(path) - - # Ensure that we have a job record for each new job that has appeared - for path in child_paths: - if path in self._bad_paths: - continue - with self._job_lock: - if path not in self._known_jobs: - # Fire off the request to populate this job asynchronously. - # - # This method is *usually* called from a asynchronous - # handler so it's better to exit from this quickly to - # allow other asynchronous handlers to be executed. - request = self._client.get_async(path) - child_proc = functools.partial(self._process_child, path) - if delayed: - request.rawlink(child_proc) - else: - child_proc(request) + for path in child_paths: + if path in self._bad_paths: + continue + # This pre-check will not guarantee that we will not already + # have the job (if it's being populated elsewhere) but it will + # reduce the amount of duplicated requests in general. + if path in self._known_jobs: + continue + if path not in investigate_paths: + investigate_paths.append(path) + for path in investigate_paths: + # Fire off the request to populate this job. + # + # This method is *usually* called from a asynchronous handler so + # it's better to exit from this quickly to allow other asynchronous + # handlers to be executed. + request = self._client.get_async(path) + if delayed: + request.rawlink(functools.partial(self._process_child, path)) + else: + self._process_child(path, request) def post(self, name, book=None, details=None): From 15e3962e2a55a35fc2e87889f1a2085163c593ce Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 16 Jul 2014 19:53:45 -0700 Subject: [PATCH 047/240] Jobboard example that show jobs + workers + producers Add a new example that spins up a set of threads to simulate the actual workers and producers that would be normally attached to a jobboard and use these threads to producer and consume a set of simple jobs (that are filtered on by the workers). - This also fixes how python3 removed the __cmp__ operator which we were using for sorting the jobs that were posted and now we must use __lt__ instead. Fixes bug 1367496 Part of blueprint more-examples Change-Id: Ib8d116637b8edae31e4c8927a28515907855f8bf --- doc/source/examples.rst | 12 ++ .../jobboard_produce_consume_colors.py | 178 ++++++++++++++++++ taskflow/jobs/backends/impl_zookeeper.py | 20 +- 3 files changed, 207 insertions(+), 3 deletions(-) create mode 100644 taskflow/examples/jobboard_produce_consume_colors.py diff --git a/doc/source/examples.rst b/doc/source/examples.rst index 89731d8e..f09fd674 100644 --- a/doc/source/examples.rst +++ b/doc/source/examples.rst @@ -198,3 +198,15 @@ Code :language: python :linenos: :lines: 16- + +Jobboard producer/consumer (simple) +=================================== + +.. note:: + + Full source located at :example:`jobboard_produce_consume_colors` + +.. literalinclude:: ../../taskflow/examples/jobboard_produce_consume_colors.py + :language: python + :linenos: + :lines: 16- diff --git a/taskflow/examples/jobboard_produce_consume_colors.py b/taskflow/examples/jobboard_produce_consume_colors.py new file mode 100644 index 00000000..7ff9265c --- /dev/null +++ b/taskflow/examples/jobboard_produce_consume_colors.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +import contextlib +import logging +import os +import random +import sys +import threading +import time + +logging.basicConfig(level=logging.ERROR) + +top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, + os.pardir)) +sys.path.insert(0, top_dir) + +import six +from zake import fake_client + +from taskflow import exceptions as excp +from taskflow.jobs import backends + +# In this example we show how a jobboard can be used to post work for other +# entities to work on. This example creates a set of jobs using one producer +# thread (typically this would be split across many machines) and then having +# other worker threads with there own jobboards select work using a given +# filters [red/blue] and then perform that work (and consuming or abandoning +# the job after it has been completed or failed). + +# Things to note: +# - No persistence layer is used (or logbook), just the job details are used +# to determine if a job should be selected by a worker or not. +# - This example runs in a single process (this is expected to be atypical +# but this example shows that it can be done if needed, for testing...) +# - The iterjobs(), claim(), consume()/abandon() worker workflow. +# - The post() producer workflow. + +SHARED_CONF = { + 'path': "/taskflow/jobs", + 'board': 'zookeeper', +} + +# How many workers and producers of work will be created (as threads). +PRODUCERS = 3 +WORKERS = 5 + +# How many units of work each producer will create. +PRODUCER_UNITS = 10 + +# How many units of work are expected to be produced (used so workers can +# know when to stop running and shutdown, typically this would not be a +# a value but we have to limit this examples execution time to be less than +# infinity). +EXPECTED_UNITS = PRODUCER_UNITS * PRODUCERS + +# Delay between producing/consuming more work. +WORKER_DELAY, PRODUCER_DELAY = (0.5, 0.5) + +# To ensure threads don't trample other threads output. +STDOUT_LOCK = threading.Lock() + + +def dispatch_work(job): + # This is where the jobs contained work *would* be done + time.sleep(1.0) + + +def safe_print(name, message, prefix=""): + with STDOUT_LOCK: + if prefix: + print("%s %s: %s" % (prefix, name, message)) + else: + print("%s: %s" % (name, message)) + + +def worker(ident, client, consumed): + # Create a personal board (using the same client so that it works in + # the same process) and start looking for jobs on the board that we want + # to perform. + name = "W-%s" % (ident) + safe_print(name, "started") + claimed_jobs = 0 + consumed_jobs = 0 + abandoned_jobs = 0 + with backends.backend(name, SHARED_CONF.copy(), client=client) as board: + while len(consumed) != EXPECTED_UNITS: + favorite_color = random.choice(['blue', 'red']) + for job in board.iterjobs(ensure_fresh=True, only_unclaimed=True): + # See if we should even bother with it... + if job.details.get('color') != favorite_color: + continue + safe_print(name, "'%s' [attempting claim]" % (job)) + try: + board.claim(job, name) + claimed_jobs += 1 + safe_print(name, "'%s' [claimed]" % (job)) + except (excp.NotFound, excp.UnclaimableJob): + safe_print(name, "'%s' [claim unsuccessful]" % (job)) + else: + try: + dispatch_work(job) + board.consume(job, name) + safe_print(name, "'%s' [consumed]" % (job)) + consumed_jobs += 1 + consumed.append(job) + except Exception: + board.abandon(job, name) + abandoned_jobs += 1 + safe_print(name, "'%s' [abandoned]" % (job)) + time.sleep(WORKER_DELAY) + safe_print(name, + "finished (claimed %s jobs, consumed %s jobs," + " abandoned %s jobs)" % (claimed_jobs, consumed_jobs, + abandoned_jobs), prefix=">>>") + + +def producer(ident, client): + # Create a personal board (using the same client so that it works in + # the same process) and start posting jobs on the board that we want + # some entity to perform. + name = "P-%s" % (ident) + safe_print(name, "started") + with backends.backend(name, SHARED_CONF.copy(), client=client) as board: + for i in six.moves.xrange(0, PRODUCER_UNITS): + job_name = "%s-%s" % (name, i) + details = { + 'color': random.choice(['red', 'blue']), + } + job = board.post(job_name, book=None, details=details) + safe_print(name, "'%s' [posted]" % (job)) + time.sleep(PRODUCER_DELAY) + safe_print(name, "finished", prefix=">>>") + + +def main(): + with contextlib.closing(fake_client.FakeClient()) as c: + created = [] + for i in range(0, PRODUCERS): + p = threading.Thread(target=producer, args=(i + 1, c)) + p.daemon = True + created.append(p) + p.start() + consumed = collections.deque() + for i in range(0, WORKERS): + w = threading.Thread(target=worker, args=(i + 1, c, consumed)) + w.daemon = True + created.append(w) + w.start() + while created: + t = created.pop() + t.join() + # At the end there should be nothing leftover, let's verify that. + board = backends.fetch('verifier', SHARED_CONF.copy(), client=c) + board.connect() + with contextlib.closing(board): + if board.job_count != 0 or len(consumed) != EXPECTED_UNITS: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/taskflow/jobs/backends/impl_zookeeper.py b/taskflow/jobs/backends/impl_zookeeper.py index 96307790..150daa42 100644 --- a/taskflow/jobs/backends/impl_zookeeper.py +++ b/taskflow/jobs/backends/impl_zookeeper.py @@ -77,10 +77,13 @@ class ZookeeperJob(base_job.Job): if all((self._book, self._book_data)): raise ValueError("Only one of 'book_data' or 'book'" " can be provided") - self._path = path + self._path = k_paths.normpath(path) self._lock_path = path + LOCK_POSTFIX self._created_on = created_on self._node_not_found = False + basename = k_paths.basename(self._path) + self._root = self._path[0:-len(basename)] + self._sequence = int(basename[len(JOB_PREFIX):]) @property def lock_path(self): @@ -90,6 +93,14 @@ class ZookeeperJob(base_job.Job): def path(self): return self._path + @property + def sequence(self): + return self._sequence + + @property + def root(self): + return self._root + def _get_node_attr(self, path, attr_name, trans_func=None): try: _data, node_stat = self._client.get(path) @@ -186,8 +197,11 @@ class ZookeeperJob(base_job.Job): return states.UNCLAIMED return states.CLAIMED - def __cmp__(self, other): - return cmp(self.path, other.path) + def __lt__(self, other): + if self.root == other.root: + return self.sequence < other.sequence + else: + return self.root < other.root def __hash__(self): return hash(self.path) From ce620c399a9e5cf2fd438d0193ba0d83da667a88 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 9 Sep 2014 18:04:40 -0700 Subject: [PATCH 048/240] Use oslotest to provide our base test case class The oslotest library has a nice openstack testing integrated base class that can ensure we setup our base test case using the right logging fixtures, test timeouts, and output fixtures that better operate in the openstack ecosystem. This will also allow us to remove some of the functionality that we currently have in our base test case and replace it with the equivalent (or better) functionality that oslotest now provides. Part of blueprint use-oslo-test Change-Id: I1602d5180ec8649a1899185972750ddddf65990f --- taskflow/test.py | 13 ++++++++++--- taskflow/tests/unit/jobs/base.py | 2 +- taskflow/tests/unit/test_duration.py | 3 +-- taskflow/tests/unit/test_engine_helpers.py | 3 +-- taskflow/tests/unit/test_storage.py | 3 +-- taskflow/tests/unit/test_task.py | 3 +-- taskflow/tests/unit/worker_based/test_dispatcher.py | 2 +- taskflow/tests/unit/worker_based/test_engine.py | 3 +-- taskflow/tests/unit/worker_based/test_executor.py | 2 +- .../tests/unit/worker_based/test_message_pump.py | 3 +-- taskflow/tests/unit/worker_based/test_protocol.py | 2 +- taskflow/tests/unit/worker_based/test_proxy.py | 2 +- taskflow/tests/unit/worker_based/test_server.py | 2 +- taskflow/tests/unit/worker_based/test_worker.py | 3 +-- test-requirements.txt | 5 +---- tox.ini | 5 +++++ 16 files changed, 29 insertions(+), 27 deletions(-) diff --git a/taskflow/test.py b/taskflow/test.py index 4de61d3e..3d94df5a 100644 --- a/taskflow/test.py +++ b/taskflow/test.py @@ -15,8 +15,15 @@ # under the License. import fixtures -import mock +from oslotest import base import six +try: + from six.moves import mock +except ImportError: + try: + from unittest import mock + except ImportError: + import mock from testtools import compat from testtools import matchers from testtools import testcase @@ -85,7 +92,7 @@ class ItemsEqual(object): return None -class TestCase(testcase.TestCase): +class TestCase(base.BaseTestCase): """Test case base class for all taskflow unit tests.""" def makeTmpDir(self): @@ -182,7 +189,7 @@ class TestCase(testcase.TestCase): self.assertThat(seq2, matcher) -class MockTestCase(TestCase): +class MockTestCase(base.BaseTestCase): def setUp(self): super(MockTestCase, self).setUp() diff --git a/taskflow/tests/unit/jobs/base.py b/taskflow/tests/unit/jobs/base.py index a178a8af..f8a2687e 100644 --- a/taskflow/tests/unit/jobs/base.py +++ b/taskflow/tests/unit/jobs/base.py @@ -19,12 +19,12 @@ import threading import time from kazoo.recipe import watchers -import mock from taskflow import exceptions as excp from taskflow.openstack.common import uuidutils from taskflow.persistence.backends import impl_dir from taskflow import states +from taskflow.test import mock from taskflow.utils import misc from taskflow.utils import persistence_utils as p_utils diff --git a/taskflow/tests/unit/test_duration.py b/taskflow/tests/unit/test_duration.py index e1588eb2..47389724 100644 --- a/taskflow/tests/unit/test_duration.py +++ b/taskflow/tests/unit/test_duration.py @@ -17,8 +17,6 @@ import contextlib import time -import mock - import taskflow.engines from taskflow import exceptions as exc from taskflow.listeners import timing @@ -26,6 +24,7 @@ from taskflow.patterns import linear_flow as lf from taskflow.persistence.backends import impl_memory from taskflow import task from taskflow import test +from taskflow.test import mock from taskflow.tests import utils as t_utils from taskflow.utils import persistence_utils as p_utils diff --git a/taskflow/tests/unit/test_engine_helpers.py b/taskflow/tests/unit/test_engine_helpers.py index fbf1756c..2353d77d 100644 --- a/taskflow/tests/unit/test_engine_helpers.py +++ b/taskflow/tests/unit/test_engine_helpers.py @@ -14,12 +14,11 @@ # License for the specific language governing permissions and limitations # under the License. -import mock - import taskflow.engines from taskflow import exceptions as exc from taskflow.patterns import linear_flow from taskflow import test +from taskflow.test import mock from taskflow.tests import utils as test_utils from taskflow.utils import persistence_utils as p_utils diff --git a/taskflow/tests/unit/test_storage.py b/taskflow/tests/unit/test_storage.py index 001cba97..94d73012 100644 --- a/taskflow/tests/unit/test_storage.py +++ b/taskflow/tests/unit/test_storage.py @@ -17,8 +17,6 @@ import contextlib import threading -import mock - from taskflow import exceptions from taskflow.openstack.common import uuidutils from taskflow.persistence import backends @@ -26,6 +24,7 @@ from taskflow.persistence import logbook from taskflow import states from taskflow import storage from taskflow import test +from taskflow.test import mock from taskflow.utils import misc from taskflow.utils import persistence_utils as p_utils diff --git a/taskflow/tests/unit/test_task.py b/taskflow/tests/unit/test_task.py index bcb75788..a3854d26 100644 --- a/taskflow/tests/unit/test_task.py +++ b/taskflow/tests/unit/test_task.py @@ -14,10 +14,9 @@ # License for the specific language governing permissions and limitations # under the License. -import mock - from taskflow import task from taskflow import test +from taskflow.test import mock from taskflow.utils import reflection diff --git a/taskflow/tests/unit/worker_based/test_dispatcher.py b/taskflow/tests/unit/worker_based/test_dispatcher.py index 4dae910d..9b121d53 100644 --- a/taskflow/tests/unit/worker_based/test_dispatcher.py +++ b/taskflow/tests/unit/worker_based/test_dispatcher.py @@ -15,10 +15,10 @@ # under the License. from kombu import message -import mock from taskflow.engines.worker_based import dispatcher from taskflow import test +from taskflow.test import mock def mock_acked_message(ack_ok=True, **kwargs): diff --git a/taskflow/tests/unit/worker_based/test_engine.py b/taskflow/tests/unit/worker_based/test_engine.py index 531dfe5a..ea322de1 100644 --- a/taskflow/tests/unit/worker_based/test_engine.py +++ b/taskflow/tests/unit/worker_based/test_engine.py @@ -14,11 +14,10 @@ # License for the specific language governing permissions and limitations # under the License. -import mock - from taskflow.engines.worker_based import engine from taskflow.patterns import linear_flow as lf from taskflow import test +from taskflow.test import mock from taskflow.tests import utils from taskflow.utils import persistence_utils as pu diff --git a/taskflow/tests/unit/worker_based/test_executor.py b/taskflow/tests/unit/worker_based/test_executor.py index cc184df2..c55c1c6a 100644 --- a/taskflow/tests/unit/worker_based/test_executor.py +++ b/taskflow/tests/unit/worker_based/test_executor.py @@ -18,12 +18,12 @@ import threading import time from concurrent import futures -import mock from oslo.utils import timeutils from taskflow.engines.worker_based import executor from taskflow.engines.worker_based import protocol as pr from taskflow import test +from taskflow.test import mock from taskflow.tests import utils from taskflow.utils import misc diff --git a/taskflow/tests/unit/worker_based/test_message_pump.py b/taskflow/tests/unit/worker_based/test_message_pump.py index 10116c21..e2102b76 100644 --- a/taskflow/tests/unit/worker_based/test_message_pump.py +++ b/taskflow/tests/unit/worker_based/test_message_pump.py @@ -16,12 +16,11 @@ import threading -import mock - from taskflow.engines.worker_based import protocol as pr from taskflow.engines.worker_based import proxy from taskflow.openstack.common import uuidutils from taskflow import test +from taskflow.test import mock from taskflow.tests import utils as test_utils from taskflow.types import latch diff --git a/taskflow/tests/unit/worker_based/test_protocol.py b/taskflow/tests/unit/worker_based/test_protocol.py index 7d51da31..00a208e2 100644 --- a/taskflow/tests/unit/worker_based/test_protocol.py +++ b/taskflow/tests/unit/worker_based/test_protocol.py @@ -15,12 +15,12 @@ # under the License. from concurrent import futures -import mock from taskflow.engines.worker_based import protocol as pr from taskflow import exceptions as excp from taskflow.openstack.common import uuidutils from taskflow import test +from taskflow.test import mock from taskflow.tests import utils from taskflow.utils import misc diff --git a/taskflow/tests/unit/worker_based/test_proxy.py b/taskflow/tests/unit/worker_based/test_proxy.py index e2dc02e8..3c075969 100644 --- a/taskflow/tests/unit/worker_based/test_proxy.py +++ b/taskflow/tests/unit/worker_based/test_proxy.py @@ -17,7 +17,7 @@ import socket import threading -import mock +from six.moves import mock from taskflow.engines.worker_based import proxy from taskflow import test diff --git a/taskflow/tests/unit/worker_based/test_server.py b/taskflow/tests/unit/worker_based/test_server.py index 2a64c960..2759c49f 100644 --- a/taskflow/tests/unit/worker_based/test_server.py +++ b/taskflow/tests/unit/worker_based/test_server.py @@ -14,13 +14,13 @@ # License for the specific language governing permissions and limitations # under the License. -import mock import six from taskflow.engines.worker_based import endpoint as ep from taskflow.engines.worker_based import protocol as pr from taskflow.engines.worker_based import server from taskflow import test +from taskflow.test import mock from taskflow.tests import utils from taskflow.utils import misc diff --git a/taskflow/tests/unit/worker_based/test_worker.py b/taskflow/tests/unit/worker_based/test_worker.py index c66255f9..17770986 100644 --- a/taskflow/tests/unit/worker_based/test_worker.py +++ b/taskflow/tests/unit/worker_based/test_worker.py @@ -14,11 +14,10 @@ # License for the specific language governing permissions and limitations # under the License. -import mock - from taskflow.engines.worker_based import endpoint from taskflow.engines.worker_based import worker from taskflow import test +from taskflow.test import mock from taskflow.tests import utils from taskflow.utils import reflection diff --git a/test-requirements.txt b/test-requirements.txt index b72ca0b8..d74dd5ee 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,11 +3,8 @@ # process, which may cause wedges in the gate later. hacking>=0.9.2,<0.10 -discover -coverage>=3.6 +oslotest>=1.1.0 # Apache-2.0 mock>=1.0 -python-subunit>=0.0.18 -testrepository>=0.0.18 testtools>=0.9.34 # Used for testing the WBE engine. diff --git a/tox.ini b/tox.ini index bc912778..3a14bd5f 100644 --- a/tox.ini +++ b/tox.ini @@ -54,6 +54,11 @@ ignore = H904 builtins = _ exclude = .venv,.tox,dist,doc,./taskflow/openstack/common,*egg,.git,build,tools +[hacking] +import_exceptions = six.moves.mock + taskflow.test.mock + unittest.mock + # NOTE(imelnikov): pyXY envs are considered to be default, so they must have # richest set of test requirements [testenv:py26] From 5780a5d77e70443dcf785c03814cf902c6d073cd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 10 Sep 2014 16:39:30 -0700 Subject: [PATCH 049/240] Use the features that the oslotest mock base class provides Instead of having our own mock subclass that has similar functions as the oslotest base mocking class just use the base class functions where we can and add on our own customizations as we choose to. This change moves to using the base classes fixtures and also adjusts the customized subclass method names to match closer to the rest of the unittest classes method name style (camel-case not underscores). Change-Id: If24530c0381d7fb99797acaa582d3be1d7054185 --- taskflow/test.py | 24 +++++++++---------- .../unit/worker_based/test_dispatcher.py | 2 +- .../tests/unit/worker_based/test_engine.py | 2 +- .../tests/unit/worker_based/test_executor.py | 8 +++---- .../unit/worker_based/test_message_pump.py | 2 +- .../tests/unit/worker_based/test_pipeline.py | 2 +- .../tests/unit/worker_based/test_proxy.py | 16 ++++++------- .../tests/unit/worker_based/test_server.py | 6 ++--- .../tests/unit/worker_based/test_worker.py | 8 +++---- 9 files changed, 34 insertions(+), 36 deletions(-) diff --git a/taskflow/test.py b/taskflow/test.py index 3d94df5a..f894b3c3 100644 --- a/taskflow/test.py +++ b/taskflow/test.py @@ -16,6 +16,7 @@ import fixtures from oslotest import base +from oslotest import mockpatch import six try: from six.moves import mock @@ -189,25 +190,23 @@ class TestCase(base.BaseTestCase): self.assertThat(seq2, matcher) -class MockTestCase(base.BaseTestCase): +class MockTestCase(TestCase): def setUp(self): super(MockTestCase, self).setUp() self.master_mock = mock.Mock(name='master_mock') - def _patch(self, target, autospec=True, **kwargs): + def patch(self, target, autospec=True, **kwargs): """Patch target and attach it to the master mock.""" - patcher = mock.patch(target, autospec=autospec, **kwargs) - mocked = patcher.start() - self.addCleanup(patcher.stop) - + f = self.useFixture(mockpatch.Patch(target, + autospec=autospec, **kwargs)) + mocked = f.mock attach_as = kwargs.pop('attach_as', None) if attach_as is not None: self.master_mock.attach_mock(mocked, attach_as) - return mocked - def _patch_class(self, module, name, autospec=True, attach_as=None): + def patchClass(self, module, name, autospec=True, attach_as=None): """Patches a modules class. This will create a class instance mock (using the provided name to @@ -219,9 +218,9 @@ class MockTestCase(base.BaseTestCase): else: instance_mock = mock.Mock() - patcher = mock.patch.object(module, name, autospec=autospec) - class_mock = patcher.start() - self.addCleanup(patcher.stop) + f = self.useFixture(mockpatch.PatchObject(module, name, + autospec=autospec)) + class_mock = f.mock class_mock.return_value = instance_mock if attach_as is None: @@ -233,8 +232,7 @@ class MockTestCase(base.BaseTestCase): self.master_mock.attach_mock(class_mock, attach_class_as) self.master_mock.attach_mock(instance_mock, attach_instance_as) - return class_mock, instance_mock - def _reset_master_mock(self): + def resetMasterMock(self): self.master_mock.reset_mock() diff --git a/taskflow/tests/unit/worker_based/test_dispatcher.py b/taskflow/tests/unit/worker_based/test_dispatcher.py index 9b121d53..a7bf2d50 100644 --- a/taskflow/tests/unit/worker_based/test_dispatcher.py +++ b/taskflow/tests/unit/worker_based/test_dispatcher.py @@ -34,7 +34,7 @@ def mock_acked_message(ack_ok=True, **kwargs): return msg -class TestDispatcher(test.MockTestCase): +class TestDispatcher(test.TestCase): def test_creation(self): on_hello = mock.MagicMock() handlers = {'hello': on_hello} diff --git a/taskflow/tests/unit/worker_based/test_engine.py b/taskflow/tests/unit/worker_based/test_engine.py index ea322de1..f274a829 100644 --- a/taskflow/tests/unit/worker_based/test_engine.py +++ b/taskflow/tests/unit/worker_based/test_engine.py @@ -31,7 +31,7 @@ class TestWorkerBasedActionEngine(test.MockTestCase): self.topics = ['test-topic1', 'test-topic2'] # patch classes - self.executor_mock, self.executor_inst_mock = self._patch_class( + self.executor_mock, self.executor_inst_mock = self.patchClass( engine.executor, 'WorkerTaskExecutor', attach_as='executor') def test_creation_default(self): diff --git a/taskflow/tests/unit/worker_based/test_executor.py b/taskflow/tests/unit/worker_based/test_executor.py index c55c1c6a..59681bb2 100644 --- a/taskflow/tests/unit/worker_based/test_executor.py +++ b/taskflow/tests/unit/worker_based/test_executor.py @@ -45,9 +45,9 @@ class TestWorkerTaskExecutor(test.MockTestCase): self.proxy_started_event = threading.Event() # patch classes - self.proxy_mock, self.proxy_inst_mock = self._patch_class( + self.proxy_mock, self.proxy_inst_mock = self.patchClass( executor.proxy, 'Proxy') - self.request_mock, self.request_inst_mock = self._patch_class( + self.request_mock, self.request_inst_mock = self.patchClass( executor.pr, 'Request', autospec=False) # other mocking @@ -56,7 +56,7 @@ class TestWorkerTaskExecutor(test.MockTestCase): self.request_inst_mock.uuid = self.task_uuid self.request_inst_mock.expired = False self.request_inst_mock.task_cls = self.task.name - self.wait_for_any_mock = self._patch( + self.wait_for_any_mock = self.patch( 'taskflow.engines.worker_based.executor.async_utils.wait_for_any') self.message_mock = mock.MagicMock(name='message') self.message_mock.properties = {'correlation_id': self.task_uuid, @@ -78,7 +78,7 @@ class TestWorkerTaskExecutor(test.MockTestCase): executor_kwargs.update(kwargs) ex = executor.WorkerTaskExecutor(**executor_kwargs) if reset_master_mock: - self._reset_master_mock() + self.resetMasterMock() return ex def test_creation(self): diff --git a/taskflow/tests/unit/worker_based/test_message_pump.py b/taskflow/tests/unit/worker_based/test_message_pump.py index e2102b76..008ad72c 100644 --- a/taskflow/tests/unit/worker_based/test_message_pump.py +++ b/taskflow/tests/unit/worker_based/test_message_pump.py @@ -29,7 +29,7 @@ BARRIER_WAIT_TIMEOUT = 1.0 POLLING_INTERVAL = 0.01 -class TestMessagePump(test.MockTestCase): +class TestMessagePump(test.TestCase): def test_notify(self): barrier = threading.Event() diff --git a/taskflow/tests/unit/worker_based/test_pipeline.py b/taskflow/tests/unit/worker_based/test_pipeline.py index 8809785e..ae11efd2 100644 --- a/taskflow/tests/unit/worker_based/test_pipeline.py +++ b/taskflow/tests/unit/worker_based/test_pipeline.py @@ -32,7 +32,7 @@ WAIT_TIMEOUT = 1.0 POLLING_INTERVAL = 0.01 -class TestPipeline(test.MockTestCase): +class TestPipeline(test.TestCase): def _fetch_server(self, task_classes): endpoints = [] for cls in task_classes: diff --git a/taskflow/tests/unit/worker_based/test_proxy.py b/taskflow/tests/unit/worker_based/test_proxy.py index 3c075969..de5f3abc 100644 --- a/taskflow/tests/unit/worker_based/test_proxy.py +++ b/taskflow/tests/unit/worker_based/test_proxy.py @@ -34,13 +34,13 @@ class TestProxy(test.MockTestCase): self.de_period = proxy.DRAIN_EVENTS_PERIOD # patch classes - self.conn_mock, self.conn_inst_mock = self._patch_class( + self.conn_mock, self.conn_inst_mock = self.patchClass( proxy.kombu, 'Connection') - self.exchange_mock, self.exchange_inst_mock = self._patch_class( + self.exchange_mock, self.exchange_inst_mock = self.patchClass( proxy.kombu, 'Exchange') - self.queue_mock, self.queue_inst_mock = self._patch_class( + self.queue_mock, self.queue_inst_mock = self.patchClass( proxy.kombu, 'Queue') - self.producer_mock, self.producer_inst_mock = self._patch_class( + self.producer_mock, self.producer_inst_mock = self.patchClass( proxy.kombu, 'Producer') # connection mocking @@ -48,14 +48,14 @@ class TestProxy(test.MockTestCase): socket.timeout, socket.timeout, KeyboardInterrupt] # connections mocking - self.connections_mock = self._patch( + self.connections_mock = self.patch( "taskflow.engines.worker_based.proxy.kombu.connections", attach_as='connections') self.connections_mock.__getitem__().acquire().__enter__.return_value =\ self.conn_inst_mock # producers mocking - self.producers_mock = self._patch( + self.producers_mock = self.patch( "taskflow.engines.worker_based.proxy.kombu.producers", attach_as='producers') self.producers_mock.__getitem__().acquire().__enter__.return_value =\ @@ -70,7 +70,7 @@ class TestProxy(test.MockTestCase): self.master_mock.attach_mock(self.on_wait_mock, 'on_wait') # reset master mock - self._reset_master_mock() + self.resetMasterMock() def _queue_name(self, topic): return "%s_%s" % (self.exchange_name, topic) @@ -99,7 +99,7 @@ class TestProxy(test.MockTestCase): proxy_kwargs.update(kwargs) p = proxy.Proxy(**proxy_kwargs) if reset_master_mock: - self._reset_master_mock() + self.resetMasterMock() return p def test_creation(self): diff --git a/taskflow/tests/unit/worker_based/test_server.py b/taskflow/tests/unit/worker_based/test_server.py index 2759c49f..40fc29b1 100644 --- a/taskflow/tests/unit/worker_based/test_server.py +++ b/taskflow/tests/unit/worker_based/test_server.py @@ -42,9 +42,9 @@ class TestServer(test.MockTestCase): ep.Endpoint(task_cls=utils.ProgressingTask)] # patch classes - self.proxy_mock, self.proxy_inst_mock = self._patch_class( + self.proxy_mock, self.proxy_inst_mock = self.patchClass( server.proxy, 'Proxy') - self.response_mock, self.response_inst_mock = self._patch_class( + self.response_mock, self.response_inst_mock = self.patchClass( server.pr, 'Response') # other mocking @@ -66,7 +66,7 @@ class TestServer(test.MockTestCase): server_kwargs.update(kwargs) s = server.Server(**server_kwargs) if reset_master_mock: - self._reset_master_mock() + self.resetMasterMock() return s def make_request(self, **kwargs): diff --git a/taskflow/tests/unit/worker_based/test_worker.py b/taskflow/tests/unit/worker_based/test_worker.py index 17770986..9d08db21 100644 --- a/taskflow/tests/unit/worker_based/test_worker.py +++ b/taskflow/tests/unit/worker_based/test_worker.py @@ -35,13 +35,13 @@ class TestWorker(test.MockTestCase): self.endpoint_count = 21 # patch classes - self.executor_mock, self.executor_inst_mock = self._patch_class( + self.executor_mock, self.executor_inst_mock = self.patchClass( worker.futures, 'ThreadPoolExecutor', attach_as='executor') - self.server_mock, self.server_inst_mock = self._patch_class( + self.server_mock, self.server_inst_mock = self.patchClass( worker.server, 'Server') # other mocking - self.threads_count_mock = self._patch( + self.threads_count_mock = self.patch( 'taskflow.engines.worker_based.worker.tu.get_optimal_thread_count') self.threads_count_mock.return_value = self.threads_count @@ -53,7 +53,7 @@ class TestWorker(test.MockTestCase): worker_kwargs.update(kwargs) w = worker.Worker(**worker_kwargs) if reset_master_mock: - self._reset_master_mock() + self.resetMasterMock() return w def test_creation(self): From 28b2f8fb1bd6a618fe4f24ed9eb0d5b6ae76d12e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 11 Sep 2014 17:07:06 -0700 Subject: [PATCH 050/240] Adjust the WBE log levels To conform better with the logging level standards move away from using LOG.exception when the level is more appropriately a warning/warn. Also changes how a message that can not be sent is really a critical error and should be treated as such (since such an error affects the overall execution model). Change-Id: I7cebd882b655958d539be36ce3b4deb75ac4a0b7 --- taskflow/engines/worker_based/executor.py | 4 +- taskflow/engines/worker_based/server.py | 37 ++++++++++++------- .../tests/unit/worker_based/test_server.py | 4 +- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index f7285c60..235f3c93 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -217,8 +217,8 @@ class WorkerTaskExecutor(executor.TaskExecutorBase): correlation_id=request.uuid) except Exception: with misc.capture_failure() as failure: - LOG.warn("Failed to submit '%s' (transitioning it to" - " %s)", request, pr.FAILURE, exc_info=True) + LOG.critical("Failed to submit '%s' (transitioning it to" + " %s)", request, pr.FAILURE, exc_info=True) if request.transition_and_log_error(pr.FAILURE, logger=LOG): del self._requests_cache[request.uuid] request.set_result(failure) diff --git a/taskflow/engines/worker_based/server.py b/taskflow/engines/worker_based/server.py index 73625865..f1a6b703 100644 --- a/taskflow/engines/worker_based/server.py +++ b/taskflow/engines/worker_based/server.py @@ -100,7 +100,6 @@ class Server(object): except KeyError: raise ValueError("The '%s' message property is missing" % prop) - return properties def _reply(self, reply_to, task_uuid, state=pr.FAILURE, **kwargs): @@ -109,7 +108,9 @@ class Server(object): try: self._proxy.publish(response, reply_to, correlation_id=task_uuid) except Exception: - LOG.exception("Failed to send reply") + LOG.critical("Failed to send reply to '%s' for task '%s' with" + " response %s", reply_to, task_uuid, response, + exc_info=True) def _on_update_progress(self, reply_to, task_uuid, task, event_data, progress): @@ -119,11 +120,13 @@ class Server(object): def _process_notify(self, notify, message): """Process notify message and reply back.""" - LOG.debug("Start processing notify message.") + LOG.debug("Started processing notify message %r", message.delivery_tag) try: reply_to = message.properties['reply_to'] - except Exception: - LOG.exception("The 'reply_to' message property is missing.") + except KeyError: + LOG.warn("The 'reply_to' message property is missing" + " in received notify message %r", message.delivery_tag, + exc_info=True) else: self._proxy.publish( msg=pr.Notify(topic=self._topic, tasks=self._endpoints.keys()), @@ -132,13 +135,17 @@ class Server(object): def _process_request(self, request, message): """Process request message and reply back.""" - # NOTE(skudriashev): parse broker message first to get the `reply_to` - # and the `task_uuid` parameters to have possibility to reply back. - LOG.debug("Start processing request message.") + LOG.debug("Started processing request message %r", + message.delivery_tag) try: + # NOTE(skudriashev): parse broker message first to get + # the `reply_to` and the `task_uuid` parameters to have + # possibility to reply back (if we can't parse, we can't respond + # in the first place...). reply_to, task_uuid = self._parse_message(message) except ValueError: - LOG.exception("Failed to parse broker message") + LOG.warn("Failed to parse request attributes from message %r", + message.delivery_tag, exc_info=True) return else: # prepare task progress callback @@ -155,7 +162,8 @@ class Server(object): progress_callback=progress_callback) except ValueError: with misc.capture_failure() as failure: - LOG.exception("Failed to parse request") + LOG.warn("Failed to parse request contents from message %r", + message.delivery_tag, exc_info=True) reply_callback(result=failure.to_dict()) return @@ -164,8 +172,9 @@ class Server(object): endpoint = self._endpoints[task_cls] except KeyError: with misc.capture_failure() as failure: - LOG.exception("The '%s' task endpoint does not exist", - task_cls) + LOG.warn("The '%s' task endpoint does not exist, unable" + " to continue processing request message %r", + task_cls, message.delivery_tag, exc_info=True) reply_callback(result=failure.to_dict()) return else: @@ -176,7 +185,9 @@ class Server(object): result = getattr(endpoint, action)(**action_args) except Exception: with misc.capture_failure() as failure: - LOG.exception("The %s task execution failed", endpoint) + LOG.warn("The '%s' endpoint '%s' execution for request" + " message %r failed", endpoint, action, + message.delivery_tag, exc_info=True) reply_callback(result=failure.to_dict()) else: if isinstance(result, misc.Failure): diff --git a/taskflow/tests/unit/worker_based/test_server.py b/taskflow/tests/unit/worker_based/test_server.py index 2a64c960..7544b5c6 100644 --- a/taskflow/tests/unit/worker_based/test_server.py +++ b/taskflow/tests/unit/worker_based/test_server.py @@ -145,7 +145,7 @@ class TestServer(test.MockTestCase): failures=dict((i, utils.FailureMatcher(f)) for i, f in six.iteritems(failures))))) - @mock.patch("taskflow.engines.worker_based.server.LOG.exception") + @mock.patch("taskflow.engines.worker_based.server.LOG.critical") def test_reply_publish_failure(self, mocked_exception): self.proxy_inst_mock.publish.side_effect = RuntimeError('Woot!') @@ -200,7 +200,7 @@ class TestServer(test.MockTestCase): ] self.assertEqual(self.master_mock.mock_calls, master_mock_calls) - @mock.patch("taskflow.engines.worker_based.server.LOG.exception") + @mock.patch("taskflow.engines.worker_based.server.LOG.warn") def test_process_request_parse_message_failure(self, mocked_exception): self.message_mock.properties = {} request = self.make_request() From dc688c18f13a9527ffce4d0175aa6a61450278af Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 11 Sep 2014 17:46:26 -0700 Subject: [PATCH 051/240] Increase robustness of WBE message and request processing When a notification request/response can't be processed ensure we log an error message at the same level as the other function that sends back responses. Also adds in a return boolean from the _reply message (which is used for the X number of replies to a servers task request) function and use this boolean to determine if the worker should attempt to perform the final handler call that activates the desired task. Change-Id: I7f3914c126f39c56d0d2e3dfe02f3b112391ff43 --- taskflow/engines/worker_based/server.py | 58 ++++++++++++++----- .../tests/unit/worker_based/test_server.py | 15 +++-- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/taskflow/engines/worker_based/server.py b/taskflow/engines/worker_based/server.py index f1a6b703..9440c96a 100644 --- a/taskflow/engines/worker_based/server.py +++ b/taskflow/engines/worker_based/server.py @@ -57,7 +57,6 @@ class Server(object): self._proxy = proxy.Proxy(topic, exchange, handlers, on_wait=None, **kwargs) self._topic = topic - self._executor = executor self._endpoints = dict([(endpoint.name, endpoint) for endpoint in endpoints]) @@ -102,21 +101,34 @@ class Server(object): prop) return properties - def _reply(self, reply_to, task_uuid, state=pr.FAILURE, **kwargs): - """Send reply to the `reply_to` queue.""" + def _reply(self, capture, reply_to, task_uuid, state=pr.FAILURE, **kwargs): + """Send a reply to the `reply_to` queue with the given information. + + Can capture failures to publish and if capturing will log associated + critical errors on behalf of the caller, and then returns whether the + publish worked out or did not. + """ response = pr.Response(state, **kwargs) + published = False try: self._proxy.publish(response, reply_to, correlation_id=task_uuid) + published = True except Exception: + if not capture: + raise LOG.critical("Failed to send reply to '%s' for task '%s' with" " response %s", reply_to, task_uuid, response, exc_info=True) + return published def _on_update_progress(self, reply_to, task_uuid, task, event_data, progress): """Send task update progress notification.""" - self._reply(reply_to, task_uuid, pr.PROGRESS, event_data=event_data, - progress=progress) + # NOTE(harlowja): the executor that will trigger this using the + # task notification/listener mechanism will handle logging if this + # fails, so thats why capture is 'False' is used here. + self._reply(False, reply_to, task_uuid, pr.PROGRESS, + event_data=event_data, progress=progress) def _process_notify(self, notify, message): """Process notify message and reply back.""" @@ -128,10 +140,14 @@ class Server(object): " in received notify message %r", message.delivery_tag, exc_info=True) else: - self._proxy.publish( - msg=pr.Notify(topic=self._topic, tasks=self._endpoints.keys()), - routing_key=reply_to - ) + response = pr.Notify(topic=self._topic, + tasks=self._endpoints.keys()) + try: + self._proxy.publish(response, routing_key=reply_to) + except Exception: + LOG.critical("Failed to send reply to '%s' with notify" + " response %s", reply_to, response, + exc_info=True) def _process_request(self, request, message): """Process request message and reply back.""" @@ -149,11 +165,11 @@ class Server(object): return else: # prepare task progress callback - progress_callback = functools.partial( - self._on_update_progress, reply_to, task_uuid) + progress_callback = functools.partial(self._on_update_progress, + reply_to, task_uuid) # prepare reply callback - reply_callback = functools.partial( - self._reply, reply_to, task_uuid) + reply_callback = functools.partial(self._reply, True, reply_to, + task_uuid) # parse request to get task name, action and action arguments try: @@ -178,11 +194,23 @@ class Server(object): reply_callback(result=failure.to_dict()) return else: - reply_callback(state=pr.RUNNING) + try: + handler = getattr(endpoint, action) + except AttributeError: + with misc.capture_failure() as failure: + LOG.warn("The '%s' handler does not exist on task endpoint" + " '%s', unable to continue processing request" + " message %r", action, endpoint, + message.delivery_tag, exc_info=True) + reply_callback(result=failure.to_dict()) + return + else: + if not reply_callback(state=pr.RUNNING): + return # perform task action try: - result = getattr(endpoint, action)(**action_args) + result = handler(**action_args) except Exception: with misc.capture_failure() as failure: LOG.warn("The '%s' endpoint '%s' execution for request" diff --git a/taskflow/tests/unit/worker_based/test_server.py b/taskflow/tests/unit/worker_based/test_server.py index 7544b5c6..36d18f4f 100644 --- a/taskflow/tests/unit/worker_based/test_server.py +++ b/taskflow/tests/unit/worker_based/test_server.py @@ -151,7 +151,7 @@ class TestServer(test.MockTestCase): # create server and process request s = self.server(reset_master_mock=True) - s._reply(self.reply_to, self.task_uuid) + s._reply(True, self.reply_to, self.task_uuid) self.assertEqual(self.master_mock.mock_calls, [ mock.call.Response(pr.FAILURE), @@ -160,6 +160,16 @@ class TestServer(test.MockTestCase): ]) self.assertTrue(mocked_exception.called) + def test_on_run_reply_failure(self): + request = self.make_request(task=utils.ProgressingTask(), arguments={}) + self.proxy_inst_mock.publish.side_effect = RuntimeError('Woot!') + + # create server and process request + s = self.server(reset_master_mock=True) + s._process_request(request, self.message_mock) + + self.assertEqual(1, self.proxy_inst_mock.publish.call_count) + def test_on_update_progress(self): request = self.make_request(task=utils.ProgressingTask(), arguments={}) @@ -270,9 +280,6 @@ class TestServer(test.MockTestCase): # check calls master_mock_calls = [ - mock.call.Response(pr.RUNNING), - mock.call.proxy.publish(self.response_inst_mock, self.reply_to, - correlation_id=self.task_uuid), mock.call.Response(pr.FAILURE, result=failure_dict), mock.call.proxy.publish(self.response_inst_mock, self.reply_to, From 8178e7811ca0f39fdaad2ca70fff9ffc18751534 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 27 Sep 2014 14:34:08 -0700 Subject: [PATCH 052/240] Adjust docs+venv tox environments requirements/dependencies After a prior merge these sections are no longer valid and need to be adjusted to reflect the removal of the optional requirements file. Change-Id: I2a93c8e55f0d692df6d074bd86ab1a5e6d11a03f --- tox.ini | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index bc912778..6636a928 100644 --- a/tox.ini +++ b/tox.ini @@ -21,10 +21,7 @@ commands = python setup.py testr --slowest --testr-args='{posargs}' [testenv:docs] basepython = python2.7 -deps = -r{toxinidir}/requirements-py2.txt - -r{toxinidir}/test-requirements.txt - -r{toxinidir}/optional-requirements.txt - doc8 +deps = {[testenv:py27]deps} commands = python setup.py build_sphinx doc8 doc/source @@ -46,6 +43,8 @@ deps = {[testenv:py27]deps} commands = python setup.py testr --coverage --testr-args='{posargs}' [testenv:venv] +basepython = python2.7 +deps = {[testenv:py27]deps} commands = {posargs} [flake8] From eedc3353c8c362ba6df62182e20a8bb4f6ad6afe Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 28 Jun 2014 21:46:46 -0700 Subject: [PATCH 053/240] Expose only `ensure_atom` from storage Move the storage ensuring logic from being split across the engine and the storage layer and expose only a single `ensure_atom` function that does the work instead. This also removes the access to the `ensure_task` and `ensure_retry` methods as the internals of the `ensure_atom` function is now the only location that needs to use these two functions. This reduces the need to do type specific atom checks in the non-storage components (which we want to reduce overall). Breaking change: removes the public methods named `ensure_task` and `ensure_retry` (which should not be used externally anyway) from the storage object and makes those internal/private methods instead. Change-Id: I3a0f1f0dd777a1633b4937e16b50030275c84d1d --- taskflow/engines/action_engine/engine.py | 7 +- taskflow/storage.py | 28 ++++- .../tests/unit/action_engine/test_runner.py | 2 +- taskflow/tests/unit/test_storage.py | 115 +++++++----------- .../tests/unit/worker_based/test_worker.py | 2 +- taskflow/tests/utils.py | 10 ++ 6 files changed, 81 insertions(+), 83 deletions(-) diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index 9bf62429..8e7b3cff 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -24,7 +24,6 @@ from taskflow.engines.action_engine import executor from taskflow.engines.action_engine import runtime from taskflow.engines import base from taskflow import exceptions as exc -from taskflow import retry from taskflow import states from taskflow import storage as atom_storage from taskflow.utils import lock_utils @@ -175,11 +174,7 @@ class ActionEngine(base.EngineBase): # a resuming state (and then to suspended). self._change_state(states.RESUMING) # does nothing in PENDING state for node in self._compilation.execution_graph.nodes_iter(): - version = misc.get_version_string(node) - if isinstance(node, retry.Retry): - self.storage.ensure_retry(node.name, version, node.save_as) - else: - self.storage.ensure_task(node.name, version, node.save_as) + self.storage.ensure_atom(node) if node.inject: self.storage.inject_atom_args(node.name, node.inject) self._change_state(states.SUSPENDED) # does nothing in PENDING state diff --git a/taskflow/storage.py b/taskflow/storage.py index 31a8868f..22f0edd4 100644 --- a/taskflow/storage.py +++ b/taskflow/storage.py @@ -23,7 +23,9 @@ import six from taskflow import exceptions from taskflow.openstack.common import uuidutils from taskflow.persistence import logbook +from taskflow import retry from taskflow import states +from taskflow import task from taskflow.utils import lock_utils from taskflow.utils import misc from taskflow.utils import reflection @@ -94,8 +96,25 @@ class Storage(object): with contextlib.closing(self._backend.get_connection()) as conn: functor(conn, *args, **kwargs) - def ensure_task(self, task_name, task_version=None, result_mapping=None): - """Ensure that there is taskdetail that corresponds the task. + def ensure_atom(self, atom): + """Ensure that there is an atomdetail in storage for the given atom. + + Returns uuid for the atomdetail that is/was created. + """ + if isinstance(atom, task.BaseTask): + return self._ensure_task(atom.name, + misc.get_version_string(atom), + atom.save_as) + elif isinstance(atom, retry.Retry): + return self._ensure_retry(atom.name, + misc.get_version_string(atom), + atom.save_as) + else: + raise TypeError("Object of type 'atom' expected." + " Got %s, %r." % (type(atom), atom)) + + def _ensure_task(self, task_name, task_version, result_mapping): + """Ensures there is a taskdetail that corresponds to the task info. If task does not exist, adds a record for it. Added task will have PENDING state. Sets result mapping for the task from result_mapping @@ -122,9 +141,8 @@ class Storage(object): self._set_result_mapping(task_name, result_mapping) return task_id - def ensure_retry(self, retry_name, retry_version=None, - result_mapping=None): - """Ensure that there is atom detail that corresponds the retry. + def _ensure_retry(self, retry_name, retry_version, result_mapping): + """Ensures there is a retrydetail that corresponds to the retry info. If retry does not exist, adds a record for it. Added retry will have PENDING state. Sets result mapping for the retry from diff --git a/taskflow/tests/unit/action_engine/test_runner.py b/taskflow/tests/unit/action_engine/test_runner.py index 2e18f6b6..48cddf6b 100644 --- a/taskflow/tests/unit/action_engine/test_runner.py +++ b/taskflow/tests/unit/action_engine/test_runner.py @@ -38,7 +38,7 @@ class _RunnerTestMixin(object): store = storage.SingleThreadedStorage(flow_detail) # This ensures the tasks exist in storage... for task in compilation.execution_graph: - store.ensure_task(task.name) + store.ensure_atom(task) if initial_state: store.set_flow_state(initial_state) task_notifier = misc.Notifier() diff --git a/taskflow/tests/unit/test_storage.py b/taskflow/tests/unit/test_storage.py index 94d73012..7d3b55b6 100644 --- a/taskflow/tests/unit/test_storage.py +++ b/taskflow/tests/unit/test_storage.py @@ -24,7 +24,7 @@ from taskflow.persistence import logbook from taskflow import states from taskflow import storage from taskflow import test -from taskflow.test import mock +from taskflow.tests import utils as test_utils from taskflow.utils import misc from taskflow.utils import persistence_utils as p_utils @@ -59,7 +59,7 @@ class StorageTestMixin(object): def test_non_saving_storage(self): _lb, flow_detail = p_utils.temporary_flow_detail(self.backend) s = storage.SingleThreadedStorage(flow_detail=flow_detail) - s.ensure_task('my_task') + s.ensure_atom(test_utils.NoopTask('my_task')) self.assertTrue(uuidutils.is_uuid_like(s.get_atom_uuid('my_task'))) def test_flow_name_and_uuid(self): @@ -70,14 +70,14 @@ class StorageTestMixin(object): def test_ensure_task(self): s = self._get_storage() - s.ensure_task('my task') + s.ensure_atom(test_utils.NoopTask('my task')) self.assertEqual(s.get_atom_state('my task'), states.PENDING) self.assertTrue(uuidutils.is_uuid_like(s.get_atom_uuid('my task'))) def test_get_tasks_states(self): s = self._get_storage() - s.ensure_task('my task') - s.ensure_task('my task2') + s.ensure_atom(test_utils.NoopTask('my task')) + s.ensure_atom(test_utils.NoopTask('my task2')) s.save('my task', 'foo') expected = { 'my task': (states.SUCCESS, states.EXECUTE), @@ -88,7 +88,9 @@ class StorageTestMixin(object): def test_ensure_task_flow_detail(self): _lb, flow_detail = p_utils.temporary_flow_detail(self.backend) s = self._get_storage(flow_detail) - s.ensure_task('my task', '3.11') + t = test_utils.NoopTask('my task') + t.version = (3, 11) + s.ensure_atom(t) td = flow_detail.find(s.get_atom_uuid('my task')) self.assertIsNotNone(td) self.assertEqual(td.name, 'my task') @@ -107,12 +109,12 @@ class StorageTestMixin(object): td = logbook.TaskDetail(name='my_task', uuid='42') flow_detail.add(td) s = self._get_storage(flow_detail) - s.ensure_task('my_task') + s.ensure_atom(test_utils.NoopTask('my_task')) self.assertEqual('42', s.get_atom_uuid('my_task')) def test_save_and_get(self): s = self._get_storage() - s.ensure_task('my task') + s.ensure_atom(test_utils.NoopTask('my task')) s.save('my task', 5) self.assertEqual(s.get('my task'), 5) self.assertEqual(s.fetch_all(), {}) @@ -120,7 +122,7 @@ class StorageTestMixin(object): def test_save_and_get_other_state(self): s = self._get_storage() - s.ensure_task('my task') + s.ensure_atom(test_utils.NoopTask('my task')) s.save('my task', 5, states.FAILURE) self.assertEqual(s.get('my task'), 5) self.assertEqual(s.get_atom_state('my task'), states.FAILURE) @@ -128,7 +130,7 @@ class StorageTestMixin(object): def test_save_and_get_cached_failure(self): failure = misc.Failure.from_exception(RuntimeError('Woot!')) s = self._get_storage() - s.ensure_task('my task') + s.ensure_atom(test_utils.NoopTask('my task')) s.save('my task', failure, states.FAILURE) self.assertEqual(s.get('my task'), failure) self.assertEqual(s.get_atom_state('my task'), states.FAILURE) @@ -138,7 +140,7 @@ class StorageTestMixin(object): def test_save_and_get_non_cached_failure(self): failure = misc.Failure.from_exception(RuntimeError('Woot!')) s = self._get_storage() - s.ensure_task('my task') + s.ensure_atom(test_utils.NoopTask('my task')) s.save('my task', failure, states.FAILURE) self.assertEqual(s.get('my task'), failure) s._failures['my task'] = None @@ -148,7 +150,7 @@ class StorageTestMixin(object): failure = misc.Failure.from_exception(RuntimeError('Woot!')) s = self._get_storage() - s.ensure_task('my task') + s.ensure_atom(test_utils.NoopTask('my task')) s.save('my task', failure, states.FAILURE) s.set_atom_state('my task', states.REVERTING) @@ -160,7 +162,7 @@ class StorageTestMixin(object): def test_get_failure_after_reload(self): failure = misc.Failure.from_exception(RuntimeError('Woot!')) s = self._get_storage() - s.ensure_task('my task') + s.ensure_atom(test_utils.NoopTask('my task')) s.save('my task', failure, states.FAILURE) s2 = self._get_storage(s._flowdetail) self.assertTrue(s2.has_failures()) @@ -170,12 +172,12 @@ class StorageTestMixin(object): def test_get_non_existing_var(self): s = self._get_storage() - s.ensure_task('my task') + s.ensure_atom(test_utils.NoopTask('my task')) self.assertRaises(exceptions.NotFound, s.get, 'my task') def test_reset(self): s = self._get_storage() - s.ensure_task('my task') + s.ensure_atom(test_utils.NoopTask('my task')) s.save('my task', 5) s.reset('my task') self.assertEqual(s.get_atom_state('my task'), states.PENDING) @@ -183,13 +185,13 @@ class StorageTestMixin(object): def test_reset_unknown_task(self): s = self._get_storage() - s.ensure_task('my task') + s.ensure_atom(test_utils.NoopTask('my task')) self.assertEqual(s.reset('my task'), None) def test_fetch_by_name(self): s = self._get_storage() name = 'my result' - s.ensure_task('my task', '1.0', {name: None}) + s.ensure_atom(test_utils.NoopTask('my task', provides=name)) s.save('my task', 5) self.assertEqual(s.fetch(name), 5) self.assertEqual(s.fetch_all(), {name: 5}) @@ -202,7 +204,7 @@ class StorageTestMixin(object): def test_task_metadata_update_with_none(self): s = self._get_storage() - s.ensure_task('my task') + s.ensure_atom(test_utils.NoopTask('my task')) s.update_atom_metadata('my task', None) self.assertEqual(s.get_task_progress('my task'), 0.0) s.set_task_progress('my task', 0.5) @@ -212,13 +214,13 @@ class StorageTestMixin(object): def test_default_task_progress(self): s = self._get_storage() - s.ensure_task('my task') + s.ensure_atom(test_utils.NoopTask('my task')) self.assertEqual(s.get_task_progress('my task'), 0.0) self.assertEqual(s.get_task_progress_details('my task'), None) def test_task_progress(self): s = self._get_storage() - s.ensure_task('my task') + s.ensure_atom(test_utils.NoopTask('my task')) s.set_task_progress('my task', 0.5, {'test_data': 11}) self.assertEqual(s.get_task_progress('my task'), 0.5) @@ -243,7 +245,7 @@ class StorageTestMixin(object): def test_task_progress_erase(self): s = self._get_storage() - s.ensure_task('my task') + s.ensure_atom(test_utils.NoopTask('my task')) s.set_task_progress('my task', 0.8, {}) self.assertEqual(s.get_task_progress('my task'), 0.8) @@ -252,24 +254,22 @@ class StorageTestMixin(object): def test_fetch_result_not_ready(self): s = self._get_storage() name = 'my result' - s.ensure_task('my task', result_mapping={name: None}) + s.ensure_atom(test_utils.NoopTask('my task', provides=name)) self.assertRaises(exceptions.NotFound, s.get, name) self.assertEqual(s.fetch_all(), {}) def test_save_multiple_results(self): s = self._get_storage() - result_mapping = {'foo': 0, 'bar': 1, 'whole': None} - s.ensure_task('my task', result_mapping=result_mapping) + s.ensure_atom(test_utils.NoopTask('my task', provides=['foo', 'bar'])) s.save('my task', ('spam', 'eggs')) self.assertEqual(s.fetch_all(), { 'foo': 'spam', 'bar': 'eggs', - 'whole': ('spam', 'eggs') }) def test_mapping_none(self): s = self._get_storage() - s.ensure_task('my task') + s.ensure_atom(test_utils.NoopTask('my task')) s.save('my task', 5) self.assertEqual(s.fetch_all(), {}) @@ -313,7 +313,7 @@ class StorageTestMixin(object): s = self._get_storage(threaded=True) def ensure_my_task(): - s.ensure_task('my_task', result_mapping={}) + s.ensure_atom(test_utils.NoopTask('my_task')) threads = [] for i in range(0, self.thread_count): @@ -356,7 +356,7 @@ class StorageTestMixin(object): def test_set_and_get_task_state(self): s = self._get_storage() state = states.PENDING - s.ensure_task('my task') + s.ensure_atom(test_utils.NoopTask('my task')) s.set_atom_state('my task', state) self.assertEqual(s.get_atom_state('my task'), state) @@ -367,7 +367,7 @@ class StorageTestMixin(object): def test_task_by_name(self): s = self._get_storage() - s.ensure_task('my task') + s.ensure_atom(test_utils.NoopTask('my task')) self.assertTrue(uuidutils.is_uuid_like(s.get_atom_uuid('my task'))) def test_transient_storage_fetch_all(self): @@ -422,69 +422,44 @@ class StorageTestMixin(object): s.set_flow_state(states.SUCCESS) self.assertEqual(s.get_flow_state(), states.SUCCESS) - @mock.patch.object(storage.LOG, 'warning') - def test_result_is_checked(self, mocked_warning): + def test_result_is_checked(self): s = self._get_storage() - s.ensure_task('my task', result_mapping={'result': 'key'}) + s.ensure_atom(test_utils.NoopTask('my task', provides=set(['result']))) s.save('my task', {}) - mocked_warning.assert_called_once_with( - mock.ANY, 'my task', 'key', 'result') self.assertRaisesRegexp(exceptions.NotFound, '^Unable to find result', s.fetch, 'result') - @mock.patch.object(storage.LOG, 'warning') - def test_empty_result_is_checked(self, mocked_warning): + def test_empty_result_is_checked(self): s = self._get_storage() - s.ensure_task('my task', result_mapping={'a': 0}) + s.ensure_atom(test_utils.NoopTask('my task', provides=['a'])) s.save('my task', ()) - mocked_warning.assert_called_once_with( - mock.ANY, 'my task', 0, 'a') self.assertRaisesRegexp(exceptions.NotFound, '^Unable to find result', s.fetch, 'a') - @mock.patch.object(storage.LOG, 'warning') - def test_short_result_is_checked(self, mocked_warning): + def test_short_result_is_checked(self): s = self._get_storage() - s.ensure_task('my task', result_mapping={'a': 0, 'b': 1}) + s.ensure_atom(test_utils.NoopTask('my task', provides=['a', 'b'])) s.save('my task', ['result']) - mocked_warning.assert_called_once_with( - mock.ANY, 'my task', 1, 'b') self.assertEqual(s.fetch('a'), 'result') self.assertRaisesRegexp(exceptions.NotFound, '^Unable to find result', s.fetch, 'b') - @mock.patch.object(storage.LOG, 'warning') - def test_multiple_providers_are_checked(self, mocked_warning): - s = self._get_storage() - s.ensure_task('my task', result_mapping={'result': 'key'}) - self.assertEqual(mocked_warning.mock_calls, []) - s.ensure_task('my other task', result_mapping={'result': 'key'}) - mocked_warning.assert_called_once_with( - mock.ANY, 'result') - - @mock.patch.object(storage.LOG, 'warning') - def test_multiple_providers_with_inject_are_checked(self, mocked_warning): - s = self._get_storage() - s.inject({'result': 'DONE'}) - self.assertEqual(mocked_warning.mock_calls, []) - s.ensure_task('my other task', result_mapping={'result': 'key'}) - mocked_warning.assert_called_once_with(mock.ANY, 'result') - def test_ensure_retry(self): s = self._get_storage() - s.ensure_retry('my retry') + s.ensure_atom(test_utils.NoopRetry('my retry')) history = s.get_retry_history('my retry') self.assertEqual(history, []) def test_ensure_retry_and_task_with_same_name(self): s = self._get_storage() - s.ensure_task('my retry') + s.ensure_atom(test_utils.NoopTask('my retry')) self.assertRaisesRegexp(exceptions.Duplicate, - '^Atom detail', s.ensure_retry, 'my retry') + '^Atom detail', s.ensure_atom, + test_utils.NoopRetry('my retry')) def test_save_retry_results(self): s = self._get_storage() - s.ensure_retry('my retry') + s.ensure_atom(test_utils.NoopRetry('my retry')) s.save('my retry', 'a') s.save('my retry', 'b') history = s.get_retry_history('my retry') @@ -492,7 +467,7 @@ class StorageTestMixin(object): def test_save_retry_results_with_mapping(self): s = self._get_storage() - s.ensure_retry('my retry', result_mapping={'x': 0}) + s.ensure_atom(test_utils.NoopRetry('my retry', provides=['x'])) s.save('my retry', 'a') s.save('my retry', 'b') history = s.get_retry_history('my retry') @@ -502,7 +477,7 @@ class StorageTestMixin(object): def test_cleanup_retry_history(self): s = self._get_storage() - s.ensure_retry('my retry', result_mapping={'x': 0}) + s.ensure_atom(test_utils.NoopRetry('my retry', provides=['x'])) s.save('my retry', 'a') s.save('my retry', 'b') s.cleanup_retry_history('my retry', states.REVERTED) @@ -513,7 +488,7 @@ class StorageTestMixin(object): def test_cached_retry_failure(self): failure = misc.Failure.from_exception(RuntimeError('Woot!')) s = self._get_storage() - s.ensure_retry('my retry', result_mapping={'x': 0}) + s.ensure_atom(test_utils.NoopRetry('my retry', provides=['x'])) s.save('my retry', 'a') s.save('my retry', failure, states.FAILURE) history = s.get_retry_history('my retry') @@ -528,14 +503,14 @@ class StorageTestMixin(object): def test_save_task_intention(self): s = self._get_storage() - s.ensure_task('my task') + s.ensure_atom(test_utils.NoopTask('my task')) s.set_atom_intention('my task', states.REVERT) intention = s.get_atom_intention('my task') self.assertEqual(intention, states.REVERT) def test_save_retry_intention(self): s = self._get_storage() - s.ensure_retry('my retry') + s.ensure_atom(test_utils.NoopTask('my retry')) s.set_atom_intention('my retry', states.RETRY) intention = s.get_atom_intention('my retry') self.assertEqual(intention, states.RETRY) diff --git a/taskflow/tests/unit/worker_based/test_worker.py b/taskflow/tests/unit/worker_based/test_worker.py index 9d08db21..d37e817f 100644 --- a/taskflow/tests/unit/worker_based/test_worker.py +++ b/taskflow/tests/unit/worker_based/test_worker.py @@ -32,7 +32,7 @@ class TestWorker(test.MockTestCase): self.exchange = 'test-exchange' self.topic = 'test-topic' self.threads_count = 5 - self.endpoint_count = 21 + self.endpoint_count = 22 # patch classes self.executor_mock, self.executor_inst_mock = self.patchClass( diff --git a/taskflow/tests/utils.py b/taskflow/tests/utils.py index d7c85b95..e7c3fc69 100644 --- a/taskflow/tests/utils.py +++ b/taskflow/tests/utils.py @@ -70,6 +70,16 @@ def zookeeper_available(min_version, timeout=3): kazoo_utils.finalize_client(client) +class NoopRetry(retry.AlwaysRevert): + pass + + +class NoopTask(task.Task): + + def execute(self): + pass + + class DummyTask(task.Task): def execute(self, context, *args, **kwargs): From be254eac665ae6fcbfd8bb640637e0e93d0ca9f9 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 16 Sep 2014 15:01:21 -0700 Subject: [PATCH 054/240] Use timeutils functions instead of misc.wallclock The common oslo timeutils functions can perform the same time methods using the better datetime objects than using the raw unix timestamps directly, so in order to reduce a little bit of code just use the functions that module provides instead of our own. Also adds a few more tests that validate the various runtime errors being thrown to ensure they are thrown when expected and handles the case where time goes backwards (say when ntpd updates) in a more reliable manner (by not becoming negative). Change-Id: I6153ff8379833844105545ddb21dede65a7d4d3a --- taskflow/tests/unit/test_types.py | 50 +++++++++++++++---- .../tests/unit/worker_based/test_protocol.py | 24 +++++---- taskflow/types/timing.py | 29 +++++++---- taskflow/utils/misc.py | 8 --- 4 files changed, 73 insertions(+), 38 deletions(-) diff --git a/taskflow/tests/unit/test_types.py b/taskflow/tests/unit/test_types.py index 0c13ee0f..0c2ab3b5 100644 --- a/taskflow/tests/unit/test_types.py +++ b/taskflow/tests/unit/test_types.py @@ -14,9 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. -import time - import networkx as nx +from oslo.utils import timeutils import six from taskflow import exceptions as excp @@ -122,43 +121,76 @@ class TreeTest(test.TestCase): class StopWatchTest(test.TestCase): + def setUp(self): + super(StopWatchTest, self).setUp() + timeutils.set_time_override() + self.addCleanup(timeutils.clear_time_override) + def test_no_states(self): watch = tt.StopWatch() self.assertRaises(RuntimeError, watch.stop) self.assertRaises(RuntimeError, watch.resume) + def test_bad_expiry(self): + self.assertRaises(ValueError, tt.StopWatch, -1) + + def test_backwards(self): + watch = tt.StopWatch(0.1) + watch.start() + timeutils.advance_time_seconds(0.5) + self.assertTrue(watch.expired()) + + timeutils.advance_time_seconds(-1.0) + self.assertFalse(watch.expired()) + self.assertEqual(0.0, watch.elapsed()) + def test_expiry(self): watch = tt.StopWatch(0.1) watch.start() - time.sleep(0.2) + timeutils.advance_time_seconds(0.2) self.assertTrue(watch.expired()) + def test_not_expired(self): + watch = tt.StopWatch(0.1) + watch.start() + timeutils.advance_time_seconds(0.05) + self.assertFalse(watch.expired()) + def test_no_expiry(self): watch = tt.StopWatch(0.1) - watch.start() - self.assertFalse(watch.expired()) + self.assertRaises(RuntimeError, watch.expired) def test_elapsed(self): watch = tt.StopWatch() watch.start() - time.sleep(0.2) + timeutils.advance_time_seconds(0.2) # NOTE(harlowja): Allow for a slight variation by using 0.19. self.assertGreaterEqual(0.19, watch.elapsed()) + def test_no_elapsed(self): + watch = tt.StopWatch() + self.assertRaises(RuntimeError, watch.elapsed) + + def test_no_leftover(self): + watch = tt.StopWatch() + self.assertRaises(RuntimeError, watch.leftover) + watch = tt.StopWatch(1) + self.assertRaises(RuntimeError, watch.leftover) + def test_pause_resume(self): watch = tt.StopWatch() watch.start() - time.sleep(0.05) + timeutils.advance_time_seconds(0.05) watch.stop() elapsed = watch.elapsed() - time.sleep(0.05) self.assertAlmostEqual(elapsed, watch.elapsed()) watch.resume() + timeutils.advance_time_seconds(0.05) self.assertNotEqual(elapsed, watch.elapsed()) def test_context_manager(self): with tt.StopWatch() as watch: - time.sleep(0.05) + timeutils.advance_time_seconds(0.05) self.assertGreater(0.01, watch.elapsed()) diff --git a/taskflow/tests/unit/worker_based/test_protocol.py b/taskflow/tests/unit/worker_based/test_protocol.py index 00a208e2..6df2bb41 100644 --- a/taskflow/tests/unit/worker_based/test_protocol.py +++ b/taskflow/tests/unit/worker_based/test_protocol.py @@ -15,6 +15,7 @@ # under the License. from concurrent import futures +from oslo.utils import timeutils from taskflow.engines.worker_based import protocol as pr from taskflow import exceptions as excp @@ -90,6 +91,8 @@ class TestProtocol(test.TestCase): def setUp(self): super(TestProtocol, self).setUp() + timeutils.set_time_override() + self.addCleanup(timeutils.clear_time_override) self.task = utils.DummyTask() self.task_uuid = 'task-uuid' self.task_action = 'execute' @@ -157,22 +160,21 @@ class TestProtocol(test.TestCase): failures={self.task.name: failure.to_dict()}) self.assertEqual(request.to_dict(), expected) - @mock.patch('taskflow.engines.worker_based.protocol.misc.wallclock') - def test_pending_not_expired(self, mocked_wallclock): - mocked_wallclock.side_effect = [0, self.timeout - 1] - self.assertFalse(self.request().expired) + def test_pending_not_expired(self): + req = self.request() + timeutils.advance_time_seconds(self.timeout - 1) + self.assertFalse(req.expired) - @mock.patch('taskflow.engines.worker_based.protocol.misc.wallclock') - def test_pending_expired(self, mocked_wallclock): - mocked_wallclock.side_effect = [0, self.timeout + 2] - self.assertTrue(self.request().expired) + def test_pending_expired(self): + req = self.request() + timeutils.advance_time_seconds(self.timeout + 1) + self.assertTrue(req.expired) - @mock.patch('taskflow.engines.worker_based.protocol.misc.wallclock') - def test_running_not_expired(self, mocked_wallclock): - mocked_wallclock.side_effect = [0, self.timeout + 2] + def test_running_not_expired(self): request = self.request() request.transition(pr.PENDING) request.transition(pr.RUNNING) + timeutils.advance_time_seconds(self.timeout + 1) self.assertFalse(request.expired) def test_set_result(self): diff --git a/taskflow/types/timing.py b/taskflow/types/timing.py index cd822ae7..40ccef79 100644 --- a/taskflow/types/timing.py +++ b/taskflow/types/timing.py @@ -16,7 +16,7 @@ import threading -from taskflow.utils import misc +from oslo.utils import timeutils class Timeout(object): @@ -55,7 +55,12 @@ class StopWatch(object): _STOPPED = 'STOPPED' def __init__(self, duration=None): - self._duration = duration + if duration is not None: + if duration < 0: + raise ValueError("Duration must be >= 0 and not %s" % duration) + self._duration = duration + else: + self._duration = None self._started_at = None self._stopped_at = None self._state = None @@ -63,19 +68,21 @@ class StopWatch(object): def start(self): if self._state == self._STARTED: return self - self._started_at = misc.wallclock() + self._started_at = timeutils.utcnow() self._stopped_at = None self._state = self._STARTED return self def elapsed(self): if self._state == self._STOPPED: - return float(self._stopped_at - self._started_at) + return max(0.0, float(timeutils.delta_seconds(self._started_at, + self._stopped_at))) elif self._state == self._STARTED: - return float(misc.wallclock() - self._started_at) + return max(0.0, float(timeutils.delta_seconds(self._started_at, + timeutils.utcnow()))) else: - raise RuntimeError("Can not get the elapsed time of an invalid" - " stopwatch") + raise RuntimeError("Can not get the elapsed time of a stopwatch" + " if it has not been started/stopped") def __enter__(self): self.start() @@ -96,12 +103,14 @@ class StopWatch(object): if self._state != self._STARTED: raise RuntimeError("Can not get the leftover time of a stopwatch" " that has not been started") - end_time = self._started_at + self._duration - return max(0.0, end_time - misc.wallclock()) + return max(0.0, self._duration - self.elapsed()) def expired(self): if self._duration is None: return False + if self._state is None: + raise RuntimeError("Can not check if a stopwatch has expired" + " if it has not been started/stopped") if self.elapsed() > self._duration: return True return False @@ -120,6 +129,6 @@ class StopWatch(object): if self._state != self._STARTED: raise RuntimeError("Can not stop a stopwatch that has not been" " started") - self._stopped_at = misc.wallclock() + self._stopped_at = timeutils.utcnow() self._state = self._STOPPED return self diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 74c27521..9100c438 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -28,7 +28,6 @@ import re import string import sys import threading -import time import traceback from oslo.serialization import jsonutils @@ -222,13 +221,6 @@ class cachedproperty(object): return value -def wallclock(): - # NOTE(harlowja): made into a function so that this can be easily mocked - # out if we want to alter time related functionality (for testing - # purposes). - return time.time() - - def millis_to_datetime(milliseconds): """Converts number of milliseconds (from epoch) into a datetime object.""" return datetime.datetime.fromtimestamp(float(milliseconds) / 1000) From cf1e468cf16818cf19777031fde049416eec4e56 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 17 Sep 2014 16:58:25 -0700 Subject: [PATCH 055/240] Add a more dynamic/useful logging listener Both cinder and glance are starting to share the same logic for there engine notification listener, so instead of having them copy around that code it will be much nicer if taskflow can just provide itself a more capable listener that both can share and use directly. This avoids users of taskflow having to understand more about the internals of taskflow and its associated state then they likely need to understand (which makes taskflow easier to use and less work to integrate). Relevant locations where this already exists: - https://github.com/openstack/cinder/blob/master/cinder/flow_utils.py - https://review.openstack.org/#/c/85211/ Change-Id: I98eeb180b31bd488ae0eadd730e1530d7bae1f1f --- doc/source/notifications.rst | 2 + taskflow/listeners/logging.py | 135 +++++++++++++++++- taskflow/test.py | 69 ++++++++++ taskflow/tests/unit/test_duration.py | 81 ----------- taskflow/tests/unit/test_listeners.py | 190 ++++++++++++++++++++++++++ 5 files changed, 390 insertions(+), 87 deletions(-) delete mode 100644 taskflow/tests/unit/test_duration.py create mode 100644 taskflow/tests/unit/test_listeners.py diff --git a/doc/source/notifications.rst b/doc/source/notifications.rst index 755f7b13..0fbd7a93 100644 --- a/doc/source/notifications.rst +++ b/doc/source/notifications.rst @@ -158,6 +158,8 @@ Printing and logging listeners .. autoclass:: taskflow.listeners.logging.LoggingListener +.. autoclass:: taskflow.listeners.logging.DynamicLoggingListener + .. autoclass:: taskflow.listeners.printing.PrintingListener Timing listener diff --git a/taskflow/listeners/logging.py b/taskflow/listeners/logging.py index 71bf83f5..175bc6fa 100644 --- a/taskflow/listeners/logging.py +++ b/taskflow/listeners/logging.py @@ -17,20 +17,38 @@ from __future__ import absolute_import import logging +import sys from taskflow.listeners import base +from taskflow import states from taskflow.utils import misc LOG = logging.getLogger(__name__) +if sys.version_info[0:2] == (2, 6): + _PY26 = True +else: + _PY26 = False + + +# Fixes this for python 2.6 which was missing the is enabled for method +# when a logger adapter is being used/provided, this will no longer be needed +# when we can just support python 2.7+ (which fixed the lack of this method +# on adapters). +def _isEnabledFor(logger, level): + if _PY26 and isinstance(logger, logging.LoggerAdapter): + return logger.logger.isEnabledFor(level) + return logger.isEnabledFor(level) + class LoggingListener(base.LoggingBase): """Listener that logs notifications it receives. - It listens for task and flow notifications and writes those - notifications to provided logger, or logger of its module - (``taskflow.listeners.logging``) if none provided. Log level - can also be configured, ``logging.DEBUG`` is used by default. + It listens for task and flow notifications and writes those notifications + to a provided logger, or logger of its module + (``taskflow.listeners.logging``) if none is provided. The log level + can also be configured, ``logging.DEBUG`` is used by default when none + is provided. """ def __init__(self, engine, task_listen_for=(misc.Notifier.ANY,), @@ -40,10 +58,115 @@ class LoggingListener(base.LoggingBase): super(LoggingListener, self).__init__(engine, task_listen_for=task_listen_for, flow_listen_for=flow_listen_for) - self._logger = log - if not self._logger: + if not log: self._logger = LOG + else: + self._logger = log self._level = level def _log(self, message, *args, **kwargs): self._logger.log(self._level, message, *args, **kwargs) + + +class DynamicLoggingListener(base.ListenerBase): + """Listener that logs notifications it receives. + + It listens for task and flow notifications and writes those notifications + to a provided logger, or logger of its module + (``taskflow.listeners.logging``) if none is provided. The log level + can *slightly* be configured and ``logging.DEBUG`` or ``logging.WARNING`` + (unless overriden via a constructor parameter) will be selected + automatically based on the execution state and results produced. + + The following flow states cause ``logging.WARNING`` (or provided + level) to be used: + + * ``states.FAILURE`` + * ``states.REVERTED`` + + The following task states cause ``logging.WARNING`` (or provided level) + to be used: + + * ``states.FAILURE`` + * ``states.RETRYING`` + * ``states.REVERTING`` + + When a task produces a :py:class:`~taskflow.utils.misc.Failure` object as + its result (typically this happens when a task raises an exception) this + will **always** switch the logger to use ``logging.WARNING`` (if the + failure object contains a ``exc_info`` tuple this will also be logged to + provide a meaningful traceback). + """ + + def __init__(self, engine, + task_listen_for=(misc.Notifier.ANY,), + flow_listen_for=(misc.Notifier.ANY,), + log=None, failure_level=logging.WARNING, + level=logging.DEBUG): + super(DynamicLoggingListener, self).__init__( + engine, + task_listen_for=task_listen_for, + flow_listen_for=flow_listen_for) + self._failure_level = failure_level + self._level = level + if not log: + self._logger = LOG + else: + self._logger = log + + def _flow_receiver(self, state, details): + # Gets called on flow state changes. + level = self._level + if state in (states.FAILURE, states.REVERTED): + level = self._failure_level + self._logger.log(level, "Flow '%s' (%s) transitioned into state '%s'" + " from state '%s'", details['flow_name'], + details['flow_uuid'], state, details.get('old_state')) + + def _task_receiver(self, state, details): + # Gets called on task state changes. + if 'result' in details and state in base.FINISH_STATES: + # If the task failed, it's useful to show the exception traceback + # and any other available exception information. + result = details.get('result') + if isinstance(result, misc.Failure): + if result.exc_info: + exc_info = result.exc_info + manual_tb = '' + else: + # When a remote failure occurs (or somehow the failure + # object lost its traceback), we will not have a valid + # exc_info that can be used but we *should* have a string + # version that we can use instead... + exc_info = None + manual_tb = "\n%s" % result.pformat(traceback=True) + self._logger.log(self._failure_level, + "Task '%s' (%s) transitioned into state" + " '%s'%s", details['task_name'], + details['task_uuid'], state, manual_tb, + exc_info=exc_info) + else: + # Otherwise, depending on the enabled logging level/state we + # will show or hide results that the task may have produced + # during execution. + level = self._level + if state == states.FAILURE: + level = self._failure_level + if (_isEnabledFor(self._logger, self._level) + or state == states.FAILURE): + self._logger.log(level, "Task '%s' (%s) transitioned into" + " state '%s' with result '%s'", + details['task_name'], + details['task_uuid'], state, + result) + else: + self._logger.log(level, "Task '%s' (%s) transitioned into" + " state '%s'", details['task_name'], + details['task_uuid'], state) + else: + level = self._level + if state in (states.REVERTING, states.RETRYING): + level = self._failure_level + self._logger.log(level, "Task '%s' (%s) transitioned into state" + " '%s'", details['task_name'], + details['task_uuid'], state) diff --git a/taskflow/test.py b/taskflow/test.py index f894b3c3..32c0b0fc 100644 --- a/taskflow/test.py +++ b/taskflow/test.py @@ -14,6 +14,9 @@ # License for the specific language governing permissions and limitations # under the License. +import collections +import logging + import fixtures from oslotest import base from oslotest import mockpatch @@ -236,3 +239,69 @@ class MockTestCase(TestCase): def resetMasterMock(self): self.master_mock.reset_mock() + + +class CapturingLoggingHandler(logging.Handler): + """A handler that saves record contents for post-test analysis.""" + + def __init__(self, level=logging.DEBUG): + # It seems needed to use the old style of base class calling, we + # can remove this old style when we only support py3.x + logging.Handler.__init__(self, level=level) + self._records = [] + + @property + def counts(self): + """Returns a dictionary with the number of records at each level.""" + self.acquire() + try: + captured = collections.defaultdict(int) + for r in self._records: + captured[r.levelno] += 1 + return captured + finally: + self.release() + + @property + def messages(self): + """Returns a dictionary with list of record messages at each level.""" + self.acquire() + try: + captured = collections.defaultdict(list) + for r in self._records: + captured[r.levelno].append(r.getMessage()) + return captured + finally: + self.release() + + @property + def exc_infos(self): + """Returns a list of all the record exc_info tuples captured.""" + self.acquire() + try: + captured = [] + for r in self._records: + if r.exc_info: + captured.append(r.exc_info) + return captured + finally: + self.release() + + def emit(self, record): + self.acquire() + try: + self._records.append(record) + finally: + self.release() + + def reset(self): + """Resets *all* internally captured state.""" + self.acquire() + try: + self._records = [] + finally: + self.release() + + def close(self): + logging.Handler.close(self) + self.reset() diff --git a/taskflow/tests/unit/test_duration.py b/taskflow/tests/unit/test_duration.py deleted file mode 100644 index 47389724..00000000 --- a/taskflow/tests/unit/test_duration.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import contextlib -import time - -import taskflow.engines -from taskflow import exceptions as exc -from taskflow.listeners import timing -from taskflow.patterns import linear_flow as lf -from taskflow.persistence.backends import impl_memory -from taskflow import task -from taskflow import test -from taskflow.test import mock -from taskflow.tests import utils as t_utils -from taskflow.utils import persistence_utils as p_utils - - -class SleepyTask(task.Task): - def __init__(self, name, sleep_for=0.0): - super(SleepyTask, self).__init__(name=name) - self._sleep_for = float(sleep_for) - - def execute(self): - if self._sleep_for <= 0: - return - else: - time.sleep(self._sleep_for) - - -class TestDuration(test.TestCase): - def make_engine(self, flow, flow_detail, backend): - e = taskflow.engines.load(flow, - flow_detail=flow_detail, - backend=backend) - e.compile() - return e - - def test_duration(self): - with contextlib.closing(impl_memory.MemoryBackend({})) as be: - flow = lf.Flow("test") - flow.add(SleepyTask("test-1", sleep_for=0.1)) - (lb, fd) = p_utils.temporary_flow_detail(be) - e = self.make_engine(flow, fd, be) - with timing.TimingListener(e): - e.run() - t_uuid = e.storage.get_atom_uuid("test-1") - td = fd.find(t_uuid) - self.assertIsNotNone(td) - self.assertIsNotNone(td.meta) - self.assertIn('duration', td.meta) - self.assertGreaterEqual(0.1, td.meta['duration']) - - @mock.patch.object(timing.LOG, 'warn') - def test_record_ending_exception(self, mocked_warn): - with contextlib.closing(impl_memory.MemoryBackend({})) as be: - flow = lf.Flow("test") - flow.add(t_utils.TaskNoRequiresNoReturns("test-1")) - (lb, fd) = p_utils.temporary_flow_detail(be) - e = self.make_engine(flow, fd, be) - timing_listener = timing.TimingListener(e) - with mock.patch.object(timing_listener._engine.storage, - 'update_atom_metadata') as mocked_uam: - mocked_uam.side_effect = exc.StorageFailure('Woot!') - with timing_listener: - e.run() - mocked_warn.assert_called_once_with(mock.ANY, mock.ANY, 'test-1', - exc_info=True) diff --git a/taskflow/tests/unit/test_listeners.py b/taskflow/tests/unit/test_listeners.py new file mode 100644 index 00000000..6ba97d6d --- /dev/null +++ b/taskflow/tests/unit/test_listeners.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import contextlib +import logging +import time + +import taskflow.engines +from taskflow import exceptions as exc +from taskflow.listeners import logging as logging_listeners +from taskflow.listeners import timing +from taskflow.patterns import linear_flow as lf +from taskflow.persistence.backends import impl_memory +from taskflow import task +from taskflow import test +from taskflow.test import mock +from taskflow.tests import utils as test_utils +from taskflow.utils import persistence_utils +from taskflow.utils import reflection + + +_LOG_LEVELS = frozenset([ + logging.CRITICAL, + logging.DEBUG, + logging.ERROR, + logging.INFO, + logging.NOTSET, + logging.WARNING, +]) + + +class SleepyTask(task.Task): + def __init__(self, name, sleep_for=0.0): + super(SleepyTask, self).__init__(name=name) + self._sleep_for = float(sleep_for) + + def execute(self): + if self._sleep_for <= 0: + return + else: + time.sleep(self._sleep_for) + + +class EngineMakerMixin(object): + def _make_engine(self, flow, flow_detail=None, backend=None): + e = taskflow.engines.load(flow, + flow_detail=flow_detail, + backend=backend) + e.compile() + e.prepare() + return e + + +class TestTimingListener(test.TestCase, EngineMakerMixin): + def test_duration(self): + with contextlib.closing(impl_memory.MemoryBackend()) as be: + flow = lf.Flow("test") + flow.add(SleepyTask("test-1", sleep_for=0.1)) + (lb, fd) = persistence_utils.temporary_flow_detail(be) + e = self._make_engine(flow, fd, be) + with timing.TimingListener(e): + e.run() + t_uuid = e.storage.get_atom_uuid("test-1") + td = fd.find(t_uuid) + self.assertIsNotNone(td) + self.assertIsNotNone(td.meta) + self.assertIn('duration', td.meta) + self.assertGreaterEqual(0.1, td.meta['duration']) + + @mock.patch.object(timing.LOG, 'warn') + def test_record_ending_exception(self, mocked_warn): + with contextlib.closing(impl_memory.MemoryBackend()) as be: + flow = lf.Flow("test") + flow.add(test_utils.TaskNoRequiresNoReturns("test-1")) + (lb, fd) = persistence_utils.temporary_flow_detail(be) + e = self._make_engine(flow, fd, be) + timing_listener = timing.TimingListener(e) + with mock.patch.object(timing_listener._engine.storage, + 'update_atom_metadata') as mocked_uam: + mocked_uam.side_effect = exc.StorageFailure('Woot!') + with timing_listener: + e.run() + mocked_warn.assert_called_once_with(mock.ANY, mock.ANY, 'test-1', + exc_info=True) + + +class TestLoggingListeners(test.TestCase, EngineMakerMixin): + def _make_logger(self, level=logging.DEBUG): + log = logging.getLogger( + reflection.get_callable_name(self._get_test_method())) + log.propagate = False + for handler in reversed(log.handlers): + log.removeHandler(handler) + handler = test.CapturingLoggingHandler(level=level) + log.addHandler(handler) + log.setLevel(level) + self.addCleanup(handler.reset) + self.addCleanup(log.removeHandler, handler) + return (log, handler) + + def test_basic(self): + flow = lf.Flow("test") + flow.add(test_utils.TaskNoRequiresNoReturns("test-1")) + e = self._make_engine(flow) + log, handler = self._make_logger() + with logging_listeners.LoggingListener(e, log=log): + e.run() + self.assertGreater(0, handler.counts[logging.DEBUG]) + for levelno in _LOG_LEVELS - set([logging.DEBUG]): + self.assertEqual(0, handler.counts[levelno]) + self.assertEqual([], handler.exc_infos) + + def test_basic_customized(self): + flow = lf.Flow("test") + flow.add(test_utils.TaskNoRequiresNoReturns("test-1")) + e = self._make_engine(flow) + log, handler = self._make_logger() + listener = logging_listeners.LoggingListener( + e, log=log, level=logging.INFO) + with listener: + e.run() + self.assertGreater(0, handler.counts[logging.INFO]) + for levelno in _LOG_LEVELS - set([logging.INFO]): + self.assertEqual(0, handler.counts[levelno]) + self.assertEqual([], handler.exc_infos) + + def test_basic_failure(self): + flow = lf.Flow("test") + flow.add(test_utils.TaskWithFailure("test-1")) + e = self._make_engine(flow) + log, handler = self._make_logger() + with logging_listeners.LoggingListener(e, log=log): + self.assertRaises(RuntimeError, e.run) + self.assertGreater(0, handler.counts[logging.DEBUG]) + for levelno in _LOG_LEVELS - set([logging.DEBUG]): + self.assertEqual(0, handler.counts[levelno]) + self.assertEqual(1, len(handler.exc_infos)) + + def test_dynamic(self): + flow = lf.Flow("test") + flow.add(test_utils.TaskNoRequiresNoReturns("test-1")) + e = self._make_engine(flow) + log, handler = self._make_logger() + with logging_listeners.DynamicLoggingListener(e, log=log): + e.run() + self.assertGreater(0, handler.counts[logging.DEBUG]) + for levelno in _LOG_LEVELS - set([logging.DEBUG]): + self.assertEqual(0, handler.counts[levelno]) + self.assertEqual([], handler.exc_infos) + + def test_dynamic_failure(self): + flow = lf.Flow("test") + flow.add(test_utils.TaskWithFailure("test-1")) + e = self._make_engine(flow) + log, handler = self._make_logger() + with logging_listeners.DynamicLoggingListener(e, log=log): + self.assertRaises(RuntimeError, e.run) + self.assertGreater(0, handler.counts[logging.WARNING]) + self.assertGreater(0, handler.counts[logging.DEBUG]) + self.assertEqual(1, len(handler.exc_infos)) + for levelno in _LOG_LEVELS - set([logging.DEBUG, logging.WARNING]): + self.assertEqual(0, handler.counts[levelno]) + + def test_dynamic_failure_customized_level(self): + flow = lf.Flow("test") + flow.add(test_utils.TaskWithFailure("test-1")) + e = self._make_engine(flow) + log, handler = self._make_logger() + listener = logging_listeners.DynamicLoggingListener( + e, log=log, failure_level=logging.ERROR) + with listener: + self.assertRaises(RuntimeError, e.run) + self.assertGreater(0, handler.counts[logging.ERROR]) + self.assertGreater(0, handler.counts[logging.DEBUG]) + self.assertEqual(1, len(handler.exc_infos)) + for levelno in _LOG_LEVELS - set([logging.DEBUG, logging.ERROR]): + self.assertEqual(0, handler.counts[levelno]) From 94b063a82a837296232998174ce9ba598b32e950 Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Mon, 29 Sep 2014 21:05:26 +0000 Subject: [PATCH 056/240] Updated from global requirements Change-Id: I48350d765a1345a98de07bc1e269940b849840b1 --- requirements-py2.txt | 4 ++-- requirements-py3.txt | 4 ++-- test-requirements.txt | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements-py2.txt b/requirements-py2.txt index 3f6292dd..61348779 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -23,5 +23,5 @@ futures>=2.1.6 jsonschema>=2.0.0,<3.0.0 # For common utilities -oslo.utils>=0.3.0 -oslo.serialization>=0.1.0 +oslo.utils>=1.0.0 # Apache-2.0 +oslo.serialization>=1.0.0 # Apache-2.0 diff --git a/requirements-py3.txt b/requirements-py3.txt index 8befdafc..ea305822 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -17,5 +17,5 @@ stevedore>=1.0.0 # Apache-2.0 jsonschema>=2.0.0,<3.0.0 # For common utilities -oslo.utils>=0.3.0 -oslo.serialization>=0.1.0 +oslo.utils>=1.0.0 # Apache-2.0 +oslo.serialization>=1.0.0 # Apache-2.0 diff --git a/test-requirements.txt b/test-requirements.txt index d74dd5ee..19856bb6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,12 +3,12 @@ # process, which may cause wedges in the gate later. hacking>=0.9.2,<0.10 -oslotest>=1.1.0 # Apache-2.0 +oslotest>=1.1.0 # Apache-2.0 mock>=1.0 testtools>=0.9.34 # Used for testing the WBE engine. -kombu>=2.4.8 +kombu>=2.5.0 # Used for testing zookeeper & backends. zake>=0.1 # Apache-2.0 From 6ccb998fb9622e0ce026500730645978a5297d31 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 28 Sep 2014 18:09:39 -0700 Subject: [PATCH 057/240] Rework the state documentation Make the state documentation easier to understand, more clear, and better stated than what existed before this commit so it is easier for users to read, use and interpret. Change-Id: I0285bfe332c6e968626719f14049a9533d066d86 --- doc/source/states.rst | 188 +++++++++++++++++++++++++----------------- 1 file changed, 111 insertions(+), 77 deletions(-) diff --git a/doc/source/states.rst b/doc/source/states.rst index 432e0079..805c10c4 100644 --- a/doc/source/states.rst +++ b/doc/source/states.rst @@ -4,6 +4,15 @@ States .. _engine states: +.. note:: + + The code contains explicit checks during transitions using the models + described below. These checks ensure that a transition is valid, if the + transition is determined to be invalid the transitioning code will raise + a :py:class:`~taskflow.exceptions.InvalidState` exception. This exception + being triggered usually means there is some kind of bug in the code or some + type of misuse/state violation is occurring, and should be reported as such. + Engine ====== @@ -41,56 +50,60 @@ Flow :align: left :alt: Flow state transitions -**PENDING** - A flow starts its life in this state. +**PENDING** - A flow starts its execution lifecycle in this state (it has no +state prior to being ran by an engine, since flow(s) are just pattern(s) +that define the semantics and ordering of their contents and flows gain +state only when they are executed). -**RUNNING** - In this state flow makes a progress, executes and/or reverts its -atoms. +**RUNNING** - In this state the engine running a flow progresses through the +flow. -**SUCCESS** - Once all atoms have finished successfully the flow transitions to -the SUCCESS state. +**SUCCESS** - Transitioned to once all of the flows atoms have finished +successfully. -**REVERTED** - The flow transitions to this state when it has been reverted -successfully after the failure. +**REVERTED** - Transitioned to once all of the flows atoms have been reverted +successfully after a failure. -**FAILURE** - The flow transitions to this state when it can not be reverted -after the failure. +**FAILURE** - The engine will transition the flow to this state when it can not +be reverted after a single failure or after multiple failures (greater than +one failure *may* occur when running in parallel). -**SUSPENDING** - In the RUNNING state the flow can be suspended. When this -happens, flow transitions to the SUSPENDING state immediately. In that state -the engine running the flow waits for running atoms to finish (since the engine -can not preempt atoms that are active). +**SUSPENDING** - In the ``RUNNING`` state the engine running the flow can be +suspended. When this happens, the engine attempts to transition the flow +to the ``SUSPENDING`` state immediately. In that state the engine running the +flow waits for running atoms to finish (since the engine can not preempt +atoms that are actively running). -**SUSPENDED** - When no atoms are running and all results received so far are -saved, the flow transitions from the SUSPENDING state to SUSPENDED. Also it may -go to the SUCCESS state if all atoms were in fact ran, or to the REVERTED state -if the flow was reverting and all atoms were reverted while the engine was -waiting for running atoms to finish, or to the FAILURE state if atoms were run -or reverted and some of them failed. - -**RESUMING** - When the flow is interrupted 'in a hard way' (e.g. server -crashed), it can be loaded from storage in any state. If the state is not -PENDING (aka, the flow was never ran) or SUCCESS, FAILURE or REVERTED (in which -case the flow has already finished), the flow gets set to the RESUMING state -for the short time period while it is being loaded from backend storage [a -database, a filesystem...] (this transition is not shown on the diagram). When -the flow is finally loaded, it goes to the SUSPENDED state. - -From the SUCCESS, FAILURE or REVERTED states the flow can be ran again (and -thus it goes back into the RUNNING state). One of the possible use cases for -this transition is to allow for alteration of a flow or flow details associated -with a previously ran flow after the flow has finished, and client code wants -to ensure that each atom from this new (potentially updated) flow has its -chance to run. +**SUSPENDED** - When no atoms are running and all results received so far have +been saved, the engine transitions the flow from the ``SUSPENDING`` state +to the ``SUSPENDED`` state. .. note:: - The current code also contains strong checks during each flow state - transition using the model described above and raises the - :py:class:`~taskflow.exceptions.InvalidState` exception if an invalid - transition is attempted. This exception being triggered usually means there - is some kind of bug in the engine code or some type of misuse/state violation - is occurring, and should be reported as such. + The engine may transition the flow to the ``SUCCESS`` state (from the + ``SUSPENDING`` state) if all atoms were in fact running (and completed) + before the suspension request was able to be honored (this is due to the lack + of preemption) or to the ``REVERTED`` state if the engine was reverting and + all atoms were reverted while the engine was waiting for running atoms to + finish or to the ``FAILURE`` state if atoms were running or reverted and + some of them had failed. +**RESUMING** - When the engine running a flow is interrupted *'in a +hard way'* (e.g. server crashed), it can be loaded from storage in *any* +state (this is required since it is can not be known what state was last +successfully saved). If the loaded state is not ``PENDING`` (aka, the flow was +never ran) or ``SUCCESS``, ``FAILURE`` or ``REVERTED`` (in which case the flow +has already finished), the flow gets set to the ``RESUMING`` state for the +short time period while it is being loaded from backend storage [a database, a +filesystem...] (this transition is not shown on the diagram). When the flow is +finally loaded, it goes to the ``SUSPENDED`` state. + +From the ``SUCCESS``, ``FAILURE`` or ``REVERTED`` states the flow can be ran +again; therefore it is allowable to go back into the ``RUNNING`` state +immediately. One of the possible use cases for this transition is to allow for +alteration of a flow or flow details associated with a previously ran flow +after the flow has finished, and client code wants to ensure that each atom +from this new (potentially updated) flow has its chance to run. Task ==== @@ -100,63 +113,84 @@ Task :align: left :alt: Task state transitions -**PENDING** - When a task is added to a flow, it starts in the PENDING state, -which means it can be executed immediately or waits for all of task it depends -on to complete. The task transitions to the PENDING state after it was -reverted and its flow was restarted or retried. +**PENDING** - A task starts its execution lifecycle in this state (it has no +state prior to being ran by an engine, since tasks(s) are just objects that +represent how to accomplish a piece of work). Once it has been transitioned to +the ``PENDING`` state by the engine this means it can be executed immediately +or if needed will wait for all of the atoms it depends on to complete. -**RUNNING** - When flow starts to execute the task, it transitions to the -RUNNING state, and stays in this state until its -:py:meth:`execute() ` method returns. +.. note:: -**SUCCESS** - The task transitions to this state after it was finished -successfully. + A engine running a task also transitions the task to the ``PENDING`` state + after it was reverted and its containing flow was restarted or retried. -**FAILURE** - The task transitions to this state after it was finished with -error. When the flow containing this task is being reverted, all its tasks are -walked in particular order. +**RUNNING** - When an engine running the task starts to execute the task, the +engine will transition the task to the ``RUNNING`` state, and the task will +stay in this state until the tasks :py:meth:`~taskflow.task.BaseTask.execute` +method returns. -**REVERTING** - The task transitions to this state when the flow starts to -revert it and its :py:meth:`revert() ` method -is called. Only tasks in the SUCCESS or FAILURE state can be reverted. If this -method fails (raises exception), the task goes to the FAILURE state. +**SUCCESS** - The engine running the task transitions the task to this state +after the task has finished successfully (ie no exception/s were raised during +execution). + +**FAILURE** - The engine running the task transitions the task to this state +after it has finished with an error. + +**REVERTING** - The engine running a task transitions the task to this state +when the containing flow the engine is running starts to revert and +its :py:meth:`~taskflow.task.BaseTask.revert` method is called. Only tasks in +the ``SUCCESS`` or ``FAILURE`` state can be reverted. If this method fails (ie +raises an exception), the task goes to the ``FAILURE`` state (if it was already +in the ``FAILURE`` state then this is a no-op). **REVERTED** - A task that has been reverted appears in this state. - Retry ===== +.. note:: + + A retry has the same states as a task and one additional state. + .. image:: img/retry_states.svg :width: 660px :align: left :alt: Retry state transitions -Retry has the same states as a task and one additional state. +**PENDING** - A retry starts its execution lifecycle in this state (it has no +state prior to being ran by an engine, since retry(s) are just objects that +represent how to retry an associated flow). Once it has been transitioned to +the ``PENDING`` state by the engine this means it can be executed immediately +or if needed will wait for all of the atoms it depends on to complete (in the +retry case the retry object will also be consulted when failures occur in the +flow that the retry is associated with by consulting its +:py:meth:`~taskflow.retry.Decider.on_failure` method). -**PENDING** - When a retry is added to a flow, it starts in the PENDING state, -which means it can be executed immediately or waits for all of task it depends -on to complete. The retry transitions to the PENDING state after it was -reverted and its flow was restarted or retried. +.. note:: -**RUNNING** - When flow starts to execute the retry, it transitions to the -RUNNING state, and stays in this state until its -:py:meth:`execute() ` method returns. + A engine running a retry also transitions the retry to the ``PENDING`` state + after it was reverted and its associated flow was restarted or retried. -**SUCCESS** - The retry transitions to this state after it was finished -successfully. +**RUNNING** - When a engine starts to execute the retry, the engine +transitions the retry to the ``RUNNING`` state, and the retry stays in this +state until its :py:meth:`~taskflow.retry.Retry.execute` method returns. -**FAILURE** - The retry transitions to this state after it was finished with -error. When the flow containing this retry is being reverted, all its tasks are -walked in particular order. +**SUCCESS** - The engine running the retry transitions it to this state after +it was finished successfully (ie no exception/s were raised during +execution). -**REVERTING** - The retry transitions to this state when the flow starts to -revert it and its :py:meth:`revert() ` method is -called. Only retries in SUCCESS or FAILURE state can be reverted. If this -method fails (raises exception), the retry goes to the FAILURE state. +**FAILURE** - The engine running the retry transitions it to this state after +it has finished with an error. + +**REVERTING** - The engine running the retry transitions to this state when +the associated flow the engine is running starts to revert it and its +:py:meth:`~taskflow.retry.Retry.revert` method is called. Only retries +in ``SUCCESS`` or ``FAILURE`` state can be reverted. If this method fails (ie +raises an exception), the retry goes to the ``FAILURE`` state (if it was +already in the ``FAILURE`` state then this is a no-op). **REVERTED** - A retry that has been reverted appears in this state. -**RETRYING** - If flow that is managed by the current retry was failed and -reverted, the engine prepares it for the next run and transitions to the -RETRYING state. +**RETRYING** - If flow that is associated with the current retry was failed and +reverted, the engine prepares the flow for the next run and transitions the +retry to the ``RETRYING`` state. From 70385ed28eb0185b4555f584762ecbcc0bba50c1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 1 Oct 2014 09:03:45 -0700 Subject: [PATCH 058/240] Make it so that the import works for older versions of kombu The import location for message was different in older versions of kombu, so when using it for mocking we need to take into account the older location so that the test correctly works out as it should (without import errors). Fixes bug 1376330 Change-Id: Ifad489f0e15490168f993f48bfedfd553acd4a69 --- taskflow/tests/unit/worker_based/test_dispatcher.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/taskflow/tests/unit/worker_based/test_dispatcher.py b/taskflow/tests/unit/worker_based/test_dispatcher.py index a7bf2d50..db0a719c 100644 --- a/taskflow/tests/unit/worker_based/test_dispatcher.py +++ b/taskflow/tests/unit/worker_based/test_dispatcher.py @@ -14,7 +14,10 @@ # License for the specific language governing permissions and limitations # under the License. -from kombu import message +try: + from kombu import message # noqa +except ImportError: + from kombu.transport import base as message from taskflow.engines.worker_based import dispatcher from taskflow import test From 95b30d60cd1be5cbdf953e30c379822aad08ab07 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 12 Sep 2014 17:29:45 -0700 Subject: [PATCH 059/240] Refactor parts of the job lock/job condition zookeeper usage This reduces the necessary locks to operations which really only need to be locked, removing some of the ones which are not needed around read only operations or operations which are thread safe when used with dictionaries (popping a single item for example) or checking if a *string* key is in a dictionary, or fetching a dictionaries length... Change-Id: I28a9d66afa7f7b733b2963b8cee3edd45696561b --- taskflow/jobs/backends/impl_zookeeper.py | 31 ++++++++++-------------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/taskflow/jobs/backends/impl_zookeeper.py b/taskflow/jobs/backends/impl_zookeeper.py index 96307790..f1decbbc 100644 --- a/taskflow/jobs/backends/impl_zookeeper.py +++ b/taskflow/jobs/backends/impl_zookeeper.py @@ -298,8 +298,7 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): self._persistence = persistence # Misc. internal details self._known_jobs = {} - self._job_lock = threading.RLock() - self._job_cond = threading.Condition(self._job_lock) + self._job_cond = threading.Condition() self._open_close_lock = threading.RLock() self._client.add_listener(self._state_change_listener) self._bad_paths = frozenset([path]) @@ -325,13 +324,12 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): @property def job_count(self): - with self._job_lock: - return len(self._known_jobs) + return len(self._known_jobs) def _fetch_jobs(self, ensure_fresh=False): if ensure_fresh: self._force_refresh() - with self._job_lock: + with self._job_cond: return sorted(six.itervalues(self._known_jobs)) def _force_refresh(self): @@ -356,8 +354,7 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): def _remove_job(self, path): LOG.debug("Removing job that was at path: %s", path) - with self._job_lock: - job = self._known_jobs.pop(path, None) + job = self._known_jobs.pop(path, None) if job is not None: self._emit(jobboard.REMOVAL, details={'job': job}) @@ -413,15 +410,14 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): child_paths.append(k_paths.join(self.path, c)) # Figure out what we really should be investigating and what we - # shouldn't... + # shouldn't (remove jobs that exist in our local version, but don't + # exist in the children anymore) and accumulate all paths that we + # need to trigger population of (without holding the job lock). investigate_paths = [] - with self._job_lock: - removals = set() + with self._job_cond: for path in six.iterkeys(self._known_jobs): if path not in child_paths: - removals.add(path) - for path in removals: - self._remove_job(path) + self._remove_job(path) for path in child_paths: if path in self._bad_paths: continue @@ -543,10 +539,9 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): if not job_path: raise ValueError("Unable to check if %r is a known path" % (job_path)) - with self._job_lock: - if job_path not in self._known_jobs: - fail_msg_tpl += ", unknown job" - raise excp.NotFound(fail_msg_tpl % (job_uuid)) + if job_path not in self._known_jobs: + fail_msg_tpl += ", unknown job" + raise excp.NotFound(fail_msg_tpl % (job_uuid)) try: yield except self._client.handler.timeout_exception as e: @@ -663,7 +658,7 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): LOG.debug("Shutting down the notifier") self._worker.shutdown() self._worker = None - with self._job_lock: + with self._job_cond: self._known_jobs.clear() LOG.debug("Stopped & cleared local state") From 27badfc314d31c6932993b117e3c12d9eb7f064f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 4 Oct 2014 13:51:51 -0700 Subject: [PATCH 060/240] Avoid usage of six.moves in local functions Currently it appears that using six.moves in threaded code isn't working as expected (something there in six does not appear to be thread safe) so until this is fixed avoid using those moves in functions in the examples and in the utility code (and instead import the moved function at the top of the module in code to avoid any threaded usage problems). Upstream bug filed at: https://bitbucket.org/gutworth/six/issue/98/ Fixes bug 1377514 Change-Id: I3fc1819df8fb42d0c3d394bbc7d047b09152af68 --- .../examples/jobboard_produce_consume_colors.py | 8 ++++---- taskflow/examples/wbe_mandelbrot.py | 14 +++++++------- taskflow/utils/kazoo_utils.py | 3 ++- taskflow/utils/misc.py | 6 ++++-- tox.ini | 2 +- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/taskflow/examples/jobboard_produce_consume_colors.py b/taskflow/examples/jobboard_produce_consume_colors.py index 7ff9265c..aa80828f 100644 --- a/taskflow/examples/jobboard_produce_consume_colors.py +++ b/taskflow/examples/jobboard_produce_consume_colors.py @@ -30,7 +30,7 @@ top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) sys.path.insert(0, top_dir) -import six +from six.moves import range as compat_range from zake import fake_client from taskflow import exceptions as excp @@ -137,7 +137,7 @@ def producer(ident, client): name = "P-%s" % (ident) safe_print(name, "started") with backends.backend(name, SHARED_CONF.copy(), client=client) as board: - for i in six.moves.xrange(0, PRODUCER_UNITS): + for i in compat_range(0, PRODUCER_UNITS): job_name = "%s-%s" % (name, i) details = { 'color': random.choice(['red', 'blue']), @@ -151,13 +151,13 @@ def producer(ident, client): def main(): with contextlib.closing(fake_client.FakeClient()) as c: created = [] - for i in range(0, PRODUCERS): + for i in compat_range(0, PRODUCERS): p = threading.Thread(target=producer, args=(i + 1, c)) p.daemon = True created.append(p) p.start() consumed = collections.deque() - for i in range(0, WORKERS): + for i in compat_range(0, WORKERS): w = threading.Thread(target=worker, args=(i + 1, c, consumed)) w.daemon = True created.append(w) diff --git a/taskflow/examples/wbe_mandelbrot.py b/taskflow/examples/wbe_mandelbrot.py index 55ca6e1a..cf46c240 100644 --- a/taskflow/examples/wbe_mandelbrot.py +++ b/taskflow/examples/wbe_mandelbrot.py @@ -20,13 +20,13 @@ import os import sys import threading -import six - top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) sys.path.insert(0, top_dir) +from six.moves import range as compat_range + from taskflow import engines from taskflow.engines.worker_based import worker from taskflow.patterns import unordered_flow as uf @@ -84,7 +84,7 @@ class MandelCalculator(task.Task): def mandelbrot(x, y, max_iters): c = complex(x, y) z = 0.0j - for i in six.moves.xrange(max_iters): + for i in compat_range(max_iters): z = z * z + c if (z.real * z.real + z.imag * z.imag) >= 4: return i @@ -95,10 +95,10 @@ class MandelCalculator(task.Task): pixel_size_x = (max_x - min_x) / width pixel_size_y = (max_y - min_y) / height block = [] - for y in six.moves.xrange(chunk[0], chunk[1]): + for y in compat_range(chunk[0], chunk[1]): row = [] imag = min_y + y * pixel_size_y - for x in six.moves.xrange(0, width): + for x in compat_range(0, width): real = min_x + x * pixel_size_x row.append(mandelbrot(real, imag, max_iters)) block.append(row) @@ -133,7 +133,7 @@ def calculate(engine_conf): # Compose our workflow. height, width = IMAGE_SIZE chunk_size = int(math.ceil(height / float(CHUNK_COUNT))) - for i in six.moves.xrange(0, CHUNK_COUNT): + for i in compat_range(0, CHUNK_COUNT): chunk_name = 'chunk_%s' % i task_name = "calculation_%s" % i # Break the calculation up into chunk size pieces. @@ -225,7 +225,7 @@ def create_fractal(): try: # Create a set of workers to simulate actual remote workers. print('Running %s workers.' % (WORKERS)) - for i in range(0, WORKERS): + for i in compat_range(0, WORKERS): worker_conf['topic'] = 'calculator_%s' % (i + 1) worker_topics.append(worker_conf['topic']) w = worker.Worker(**worker_conf) diff --git a/taskflow/utils/kazoo_utils.py b/taskflow/utils/kazoo_utils.py index ae62e880..93da2cdd 100644 --- a/taskflow/utils/kazoo_utils.py +++ b/taskflow/utils/kazoo_utils.py @@ -17,6 +17,7 @@ from kazoo import client from kazoo import exceptions as k_exc import six +from six.moves import zip as compat_zip from taskflow import exceptions as exc from taskflow.utils import reflection @@ -100,7 +101,7 @@ def checked_commit(txn): return [] results = txn.commit() failures = [] - for op, result in six.moves.zip(txn.operations, results): + for op, result in compat_zip(txn.operations, results): if isinstance(result, k_exc.KazooException): failures.append((op, result)) if len(results) < len(txn.operations): diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 74c27521..d05e68db 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -34,6 +34,8 @@ import traceback from oslo.serialization import jsonutils from oslo.utils import netutils import six +from six.moves import map as compat_map +from six.moves import range as compat_range from six.moves.urllib import parse as urlparse from taskflow import exceptions as exc @@ -286,7 +288,7 @@ def item_from(container, index, name=None): def get_duplicate_keys(iterable, key=None): if key is not None: - iterable = six.moves.map(key, iterable) + iterable = compat_map(key, iterable) keys = set() duplicates = set() for item in iterable: @@ -373,7 +375,7 @@ class ExponentialBackoff(object): def __iter__(self): if self.count <= 0: raise StopIteration() - for i in six.moves.range(0, self.count): + for i in compat_range(0, self.count): yield min(self.exponent ** i, self.max_backoff) def __str__(self): diff --git a/tox.ini b/tox.ini index d2e429f6..849820d4 100644 --- a/tox.ini +++ b/tox.ini @@ -54,7 +54,7 @@ builtins = _ exclude = .venv,.tox,dist,doc,./taskflow/openstack/common,*egg,.git,build,tools [hacking] -import_exceptions = six.moves.mock +import_exceptions = six.moves taskflow.test.mock unittest.mock From 8d143187eaa0c5554bb72b9258e2b3538e7c16ea Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 7 Oct 2014 10:56:30 -0700 Subject: [PATCH 061/240] Update engine class names to better reflect there usage Rename the single threaded engine to be the serial engine which better matches its entrypoint, do the same for the multithreaded engine (renaming it to the parallel engine). Change-Id: I6174b4f1936858c13eeee416bfa3836cf20a1350 --- doc/source/notifications.rst | 12 ++++++------ setup.cfg | 6 +++--- taskflow/engines/action_engine/engine.py | 8 ++++---- taskflow/examples/run_by_iter.py | 13 +++++-------- taskflow/examples/run_by_iter_enumerate.py | 8 ++++---- taskflow/tests/unit/test_engines.py | 6 +++--- 6 files changed, 25 insertions(+), 28 deletions(-) diff --git a/doc/source/notifications.rst b/doc/source/notifications.rst index 755f7b13..e147e902 100644 --- a/doc/source/notifications.rst +++ b/doc/source/notifications.rst @@ -137,14 +137,14 @@ For example, this is how you can use >>> with printing.PrintingListener(eng): ... eng.run() ... - taskflow.engines.action_engine.engine.SingleThreadedActionEngine: ... has moved flow 'cat-dog' (...) into state 'RUNNING' - taskflow.engines.action_engine.engine.SingleThreadedActionEngine: ... has moved task 'CatTalk' (...) into state 'RUNNING' + taskflow.engines.action_engine.engine.SerialActionEngine: ... has moved flow 'cat-dog' (...) into state 'RUNNING' + taskflow.engines.action_engine.engine.SerialActionEngine: ... has moved task 'CatTalk' (...) into state 'RUNNING' meow - taskflow.engines.action_engine.engine.SingleThreadedActionEngine: ... has moved task 'CatTalk' (...) into state 'SUCCESS' with result 'cat' (failure=False) - taskflow.engines.action_engine.engine.SingleThreadedActionEngine: ... has moved task 'DogTalk' (...) into state 'RUNNING' + taskflow.engines.action_engine.engine.SerialActionEngine: ... has moved task 'CatTalk' (...) into state 'SUCCESS' with result 'cat' (failure=False) + taskflow.engines.action_engine.engine.SerialActionEngine: ... has moved task 'DogTalk' (...) into state 'RUNNING' woof - taskflow.engines.action_engine.engine.SingleThreadedActionEngine: ... has moved task 'DogTalk' (...) into state 'SUCCESS' with result 'dog' (failure=False) - taskflow.engines.action_engine.engine.SingleThreadedActionEngine: ... has moved flow 'cat-dog' (...) into state 'SUCCESS' + taskflow.engines.action_engine.engine.SerialActionEngine: ... has moved task 'DogTalk' (...) into state 'SUCCESS' with result 'dog' (failure=False) + taskflow.engines.action_engine.engine.SerialActionEngine: ... has moved flow 'cat-dog' (...) into state 'SUCCESS' Basic listener -------------- diff --git a/setup.cfg b/setup.cfg index 0c396204..dadce717 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,9 +49,9 @@ taskflow.persistence = zookeeper = taskflow.persistence.backends.impl_zookeeper:ZkBackend taskflow.engines = - default = taskflow.engines.action_engine.engine:SingleThreadedActionEngine - serial = taskflow.engines.action_engine.engine:SingleThreadedActionEngine - parallel = taskflow.engines.action_engine.engine:MultiThreadedActionEngine + default = taskflow.engines.action_engine.engine:SerialActionEngine + serial = taskflow.engines.action_engine.engine:SerialActionEngine + parallel = taskflow.engines.action_engine.engine:ParallelActionEngine worker-based = taskflow.engines.worker_based.engine:WorkerBasedActionEngine [nosetests] diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index 9bf62429..bbd89c9d 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -225,12 +225,12 @@ class ActionEngine(base.EngineBase): self._compiled = True -class SingleThreadedActionEngine(ActionEngine): +class SerialActionEngine(ActionEngine): """Engine that runs tasks in serial manner.""" _storage_factory = atom_storage.SingleThreadedStorage -class MultiThreadedActionEngine(ActionEngine): +class ParallelActionEngine(ActionEngine): """Engine that runs tasks in parallel manner.""" _storage_factory = atom_storage.MultiThreadedStorage @@ -240,7 +240,7 @@ class MultiThreadedActionEngine(ActionEngine): def __init__(self, flow, flow_detail, backend, conf, executor=None, max_workers=None): - super(MultiThreadedActionEngine, self).__init__( - flow, flow_detail, backend, conf) + super(ParallelActionEngine, self).__init__(flow, flow_detail, + backend, conf) self._executor = executor self._max_workers = max_workers diff --git a/taskflow/examples/run_by_iter.py b/taskflow/examples/run_by_iter.py index 0a7761b7..4b7b98cc 100644 --- a/taskflow/examples/run_by_iter.py +++ b/taskflow/examples/run_by_iter.py @@ -30,9 +30,9 @@ sys.path.insert(0, top_dir) sys.path.insert(0, self_dir) -from taskflow.engines.action_engine import engine +from taskflow import engines from taskflow.patterns import linear_flow as lf -from taskflow.persistence.backends import impl_memory +from taskflow.persistence import backends as persistence_backends from taskflow import task from taskflow.utils import persistence_utils @@ -73,18 +73,15 @@ flows = [] for i in range(0, flow_count): f = make_alphabet_flow(i + 1) flows.append(make_alphabet_flow(i + 1)) -be = impl_memory.MemoryBackend({}) +be = persistence_backends.fetch(conf={'connection': 'memory'}) book = persistence_utils.temporary_log_book(be) -engines = [] +engine_iters = [] for f in flows: fd = persistence_utils.create_flow_detail(f, book, be) - e = engine.SingleThreadedActionEngine(f, fd, be, {}) + e = engines.load(f, flow_detail=fd, backend=be, book=book) e.compile() e.storage.inject({'A': 'A'}) e.prepare() - engines.append(e) -engine_iters = [] -for e in engines: engine_iters.append(e.run_iter()) while engine_iters: for it in list(engine_iters): diff --git a/taskflow/examples/run_by_iter_enumerate.py b/taskflow/examples/run_by_iter_enumerate.py index 66b1859f..d954d6aa 100644 --- a/taskflow/examples/run_by_iter_enumerate.py +++ b/taskflow/examples/run_by_iter_enumerate.py @@ -27,9 +27,9 @@ top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), sys.path.insert(0, top_dir) sys.path.insert(0, self_dir) -from taskflow.engines.action_engine import engine +from taskflow import engines from taskflow.patterns import linear_flow as lf -from taskflow.persistence.backends import impl_memory +from taskflow.persistence import backends as persistence_backends from taskflow import task from taskflow.utils import persistence_utils @@ -48,10 +48,10 @@ f = lf.Flow("counter") for i in range(0, 10): f.add(EchoNameTask("echo_%s" % (i + 1))) -be = impl_memory.MemoryBackend() +be = persistence_backends.fetch(conf={'connection': 'memory'}) book = persistence_utils.temporary_log_book(be) fd = persistence_utils.create_flow_detail(f, book, be) -e = engine.SingleThreadedActionEngine(f, fd, be, {}) +e = engines.load(f, flow_detail=fd, backend=be, book=book) e.compile() e.prepare() diff --git a/taskflow/tests/unit/test_engines.py b/taskflow/tests/unit/test_engines.py index d2fb0d43..243635fb 100644 --- a/taskflow/tests/unit/test_engines.py +++ b/taskflow/tests/unit/test_engines.py @@ -555,11 +555,11 @@ class SingleThreadedEngineTest(EngineTaskTest, def test_correct_load(self): engine = self._make_engine(utils.TaskNoRequiresNoReturns) - self.assertIsInstance(engine, eng.SingleThreadedActionEngine) + self.assertIsInstance(engine, eng.SerialActionEngine) def test_singlethreaded_is_the_default(self): engine = taskflow.engines.load(utils.TaskNoRequiresNoReturns) - self.assertIsInstance(engine, eng.SingleThreadedActionEngine) + self.assertIsInstance(engine, eng.SerialActionEngine) class MultiThreadedEngineTest(EngineTaskTest, @@ -578,7 +578,7 @@ class MultiThreadedEngineTest(EngineTaskTest, def test_correct_load(self): engine = self._make_engine(utils.TaskNoRequiresNoReturns) - self.assertIsInstance(engine, eng.MultiThreadedActionEngine) + self.assertIsInstance(engine, eng.ParallelActionEngine) self.assertIs(engine._executor, None) def test_using_common_executor(self): From 94b4b60967c052be2e88a78515666dafcbda7114 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 8 Oct 2014 12:36:25 -0700 Subject: [PATCH 062/240] Allow the worker banner to be written to an arbitrary location Instead of always writing the worker startup banner to LOG.info make it so that a callback can be provided that will receive the worker banner and can write it to a desired location that the callback specifies (or drop it completely). This is useful for those who use workers but have an alternate location they desire the banner to go (for example some arbitrary stream). Change-Id: I61a4b7e9dd33ee06137caaed33153f5b3fbeb661 --- taskflow/engines/worker_based/worker.py | 10 +++++++--- taskflow/tests/unit/worker_based/test_worker.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/taskflow/engines/worker_based/worker.py b/taskflow/engines/worker_based/worker.py index ee3ea159..de55a8e2 100644 --- a/taskflow/engines/worker_based/worker.py +++ b/taskflow/engines/worker_based/worker.py @@ -143,11 +143,15 @@ class Worker(object): return BANNER_TEMPLATE.substitute(BANNER_TEMPLATE.defaults, **tpl_params) - def run(self, display_banner=True): + def run(self, display_banner=True, banner_writer=None): """Runs the worker.""" if display_banner: - for line in self._generate_banner().splitlines(): - LOG.info(line) + banner = self._generate_banner() + if banner_writer is None: + for line in banner.splitlines(): + LOG.info(line) + else: + banner_writer(banner) self._server.start() def wait(self): diff --git a/taskflow/tests/unit/worker_based/test_worker.py b/taskflow/tests/unit/worker_based/test_worker.py index 9d08db21..2b50783a 100644 --- a/taskflow/tests/unit/worker_based/test_worker.py +++ b/taskflow/tests/unit/worker_based/test_worker.py @@ -14,6 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. +import six + from taskflow.engines.worker_based import endpoint from taskflow.engines.worker_based import worker from taskflow import test @@ -66,6 +68,14 @@ class TestWorker(test.MockTestCase): ] self.assertEqual(self.master_mock.mock_calls, master_mock_calls) + def test_banner_writing(self): + buf = six.StringIO() + w = self.worker() + w.run(banner_writer=buf.write) + w.wait() + w.stop() + self.assertGreater(0, len(buf.getvalue())) + def test_creation_with_custom_threads_count(self): self.worker(threads_count=10) From b86b7e15d444d38768993263bfc4861f63e348fb Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 7 Oct 2014 21:33:50 -0700 Subject: [PATCH 063/240] Switch to a custom NotImplementedError derivative When a feature or method is not implemented it's useful to throw our own derivative of a NotImplementedError error so that we can distingush these types of errors vs actual usage of NotImplementedError which could be thrown from driver or user code. Change-Id: I8d5dfb56a254f315c5509dc600a078cfef2dde0b --- taskflow/exceptions.py | 9 +++++++++ taskflow/listeners/base.py | 4 ++-- taskflow/persistence/logbook.py | 7 ++++--- taskflow/tests/unit/persistence/test_sql_persistence.py | 7 +++++-- taskflow/tests/utils.py | 4 +++- 5 files changed, 23 insertions(+), 8 deletions(-) diff --git a/taskflow/exceptions.py b/taskflow/exceptions.py index 21bf35ee..29307bbb 100644 --- a/taskflow/exceptions.py +++ b/taskflow/exceptions.py @@ -135,6 +135,15 @@ class InvalidFormat(TaskFlowException): # Others. +class NotImplementedError(NotImplementedError): + """Exception for when some functionality really isn't implemented. + + This is typically useful when the library itself needs to distinguish + internal features not being made available from users features not being + made available/implemented (and to avoid misinterpreting the two). + """ + + class WrappedFailure(Exception): """Wraps one or several failure objects. diff --git a/taskflow/listeners/base.py b/taskflow/listeners/base.py index 0b15cce4..f11202f5 100644 --- a/taskflow/listeners/base.py +++ b/taskflow/listeners/base.py @@ -124,12 +124,12 @@ class LoggingBase(ListenerBase): backend. To implement your own logging listener derive form this class and - override ``_log`` method. + override the ``_log`` method. """ @abc.abstractmethod def _log(self, message, *args, **kwargs): - raise NotImplementedError() + """Logs the provided *templated* message to some output.""" def _flow_receiver(self, state, details): self._log("%s has moved flow '%s' (%s) into state '%s'", diff --git a/taskflow/persistence/logbook.py b/taskflow/persistence/logbook.py index 974d8461..3d60aa52 100644 --- a/taskflow/persistence/logbook.py +++ b/taskflow/persistence/logbook.py @@ -411,7 +411,8 @@ class TaskDetail(AtomDetail): def merge(self, other, deep_copy=False): if not isinstance(other, TaskDetail): - raise NotImplementedError("Can only merge with other task details") + raise exc.NotImplementedError("Can only merge with other" + " task details") if other is self: return self super(TaskDetail, self).merge(other, deep_copy=deep_copy) @@ -496,8 +497,8 @@ class RetryDetail(AtomDetail): def merge(self, other, deep_copy=False): if not isinstance(other, RetryDetail): - raise NotImplementedError("Can only merge with other retry " - "details") + raise exc.NotImplementedError("Can only merge with other" + " retry details") if other is self: return self super(RetryDetail, self).merge(other, deep_copy=deep_copy) diff --git a/taskflow/tests/unit/persistence/test_sql_persistence.py b/taskflow/tests/unit/persistence/test_sql_persistence.py index 2baaa373..229ef310 100644 --- a/taskflow/tests/unit/persistence/test_sql_persistence.py +++ b/taskflow/tests/unit/persistence/test_sql_persistence.py @@ -14,11 +14,13 @@ # License for the specific language governing permissions and limitations # under the License. +import abc import contextlib import os import random import tempfile +import six import testtools @@ -136,19 +138,20 @@ class SqlitePersistenceTest(test.TestCase, base.PersistenceTestMixin): self.db_location = None +@six.add_metaclass(abc.ABCMeta) class BackendPersistenceTestMixin(base.PersistenceTestMixin): """Specifies a backend type and does required setup and teardown.""" def _get_connection(self): return self.backend.get_connection() + @abc.abstractmethod def _init_db(self): """Sets up the database, and returns the uri to that database.""" - raise NotImplementedError() + @abc.abstractmethod def _remove_db(self): """Cleans up by removing the database once the tests are done.""" - raise NotImplementedError() def setUp(self): super(BackendPersistenceTestMixin, self).setUp() diff --git a/taskflow/tests/utils.py b/taskflow/tests/utils.py index d7c85b95..9c5ca5a7 100644 --- a/taskflow/tests/utils.py +++ b/taskflow/tests/utils.py @@ -285,7 +285,9 @@ class EngineTestBase(object): super(EngineTestBase, self).tearDown() def _make_engine(self, flow, **kwargs): - raise NotImplementedError() + raise exceptions.NotImplementedError("_make_engine() must be" + " overridden if an engine is" + " desired") class FailureMatcher(object): From 52494e745ebfc548c688c48ab9e02b20222fa460 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 9 Oct 2014 13:18:02 -0700 Subject: [PATCH 064/240] Change messaging from handler connection timeouts -> operation timeouts Instead of having the exception message say the issue is a connection timeout (which it may not be if the operation just takes a long time to finish...) change the message to denote that its an operation time out instead. Change-Id: Icf20e88ca6b23719b19d98298d7869c783f32f52 --- taskflow/jobs/backends/impl_zookeeper.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/taskflow/jobs/backends/impl_zookeeper.py b/taskflow/jobs/backends/impl_zookeeper.py index 150daa42..39c397df 100644 --- a/taskflow/jobs/backends/impl_zookeeper.py +++ b/taskflow/jobs/backends/impl_zookeeper.py @@ -115,7 +115,7 @@ class ZookeeperJob(base_job.Job): % (attr_name, self.uuid, self.path, path), e) except self._client.handler.timeout_exception as e: raise excp.JobFailure("Can not fetch the %r attribute" - " of job %s (%s), connection timed out" + " of job %s (%s), operation timed out" % (attr_name, self.uuid, self.path), e) except k_exceptions.SessionExpiredError as e: raise excp.JobFailure("Can not fetch the %r attribute" @@ -183,7 +183,7 @@ class ZookeeperJob(base_job.Job): " session expired" % (self.uuid), e) except self._client.handler.timeout_exception as e: raise excp.JobFailure("Can not fetch the state of %s," - " connection timed out" % (self.uuid), e) + " operation timed out" % (self.uuid), e) except k_exceptions.KazooException as e: raise excp.JobFailure("Can not fetch the state of %s, internal" " error" % (self.uuid), e) @@ -352,7 +352,7 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): try: children = self._client.get_children(self.path) except self._client.handler.timeout_exception as e: - raise excp.JobFailure("Refreshing failure, connection timed out", + raise excp.JobFailure("Refreshing failure, operation timed out", e) except k_exceptions.SessionExpiredError as e: raise excp.JobFailure("Refreshing failure, session expired", e) @@ -386,7 +386,7 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): LOG.warn("Incorrectly formatted job data found at path: %s", path, exc_info=True) except self._client.handler.timeout_exception: - LOG.warn("Connection timed out fetching job data from path: %s", + LOG.warn("Operation timed out fetching job data from path: %s", path, exc_info=True) except k_exceptions.SessionExpiredError: LOG.warn("Session expired fetching job data from path: %s", path, @@ -564,7 +564,7 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): try: yield except self._client.handler.timeout_exception as e: - fail_msg_tpl += ", connection timed out" + fail_msg_tpl += ", operation timed out" raise excp.JobFailure(fail_msg_tpl % (job_uuid), e) except k_exceptions.SessionExpiredError as e: fail_msg_tpl += ", session expired" From c90e36020a6993166bae1013402c30d827014317 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 9 Oct 2014 17:55:36 -0700 Subject: [PATCH 065/240] Add the database schema to the sqlalchemy docs In order to show people that are reading the docs what the current schema for the database is create a helper tool that upgrades the schema to the newest version and then uses tabulate to create a restructured text version of that schema which is then included in the documentation. Change-Id: Id215f88430971a4a083f9739fb2ec59d971dc8fa --- doc/source/persistence.rst | 48 ++++++++++++++++++++++ tools/schema_generator.py | 83 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100755 tools/schema_generator.py diff --git a/doc/source/persistence.rst b/doc/source/persistence.rst index 18fe6bff..0da68de1 100644 --- a/doc/source/persistence.rst +++ b/doc/source/persistence.rst @@ -177,6 +177,54 @@ Useful when you need a higher level of durability than offered by the previous solutions. When using these connection types it is possible to resume a engine from a peer machine (this does not apply when using sqlite). +Schema +^^^^^^ + +*Logbooks* + +========== ======== ============= +Name Type Primary Key +========== ======== ============= +created_at DATETIME False +updated_at DATETIME False +uuid VARCHAR True +name VARCHAR False +meta TEXT False +========== ======== ============= + +*Flow details* + +=========== ======== ============= +Name Type Primary Key +=========== ======== ============= +created_at DATETIME False +updated_at DATETIME False +uuid VARCHAR True +name VARCHAR False +meta TEXT False +state VARCHAR False +parent_uuid VARCHAR False +=========== ======== ============= + +*Atom details* + +=========== ======== ============= +Name Type Primary Key +=========== ======== ============= +created_at DATETIME False +updated_at DATETIME False +uuid VARCHAR True +name VARCHAR False +meta TEXT False +atom_type VARCHAR False +state VARCHAR False +intention VARCHAR False +results TEXT False +failure TEXT False +version TEXT False +parent_uuid VARCHAR False +=========== ======== ============= + .. _sqlalchemy: http://www.sqlalchemy.org/docs/ .. _ACID: https://en.wikipedia.org/wiki/ACID diff --git a/tools/schema_generator.py b/tools/schema_generator.py new file mode 100755 index 00000000..3685a0a1 --- /dev/null +++ b/tools/schema_generator.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import contextlib +import re + +import six +import tabulate + +from taskflow.persistence.backends import impl_sqlalchemy + +NAME_MAPPING = { + 'flowdetails': 'Flow details', + 'atomdetails': 'Atom details', + 'logbooks': 'Logbooks', +} +CONN_CONF = { + # This uses an in-memory database (aka nothing is written) + "connection": "sqlite://", +} +TABLE_QUERY = "SELECT name, sql FROM sqlite_master WHERE type='table'" +SCHEMA_QUERY = "pragma table_info(%s)" + + +def to_bool_string(val): + if isinstance(val, (int, bool)): + return six.text_type(bool(val)) + if not isinstance(val, six.string_types): + val = six.text_type(val) + if val.lower() in ('0', 'false'): + return 'False' + if val.lower() in ('1', 'true'): + return 'True' + raise ValueError("Unknown boolean input '%s'" % (val)) + + +def main(): + backend = impl_sqlalchemy.SQLAlchemyBackend(CONN_CONF) + with contextlib.closing(backend) as backend: + # Make the schema exist... + with contextlib.closing(backend.get_connection()) as conn: + conn.upgrade() + # Now make a prettier version of that schema... + tables = backend.engine.execute(TABLE_QUERY) + table_names = [r[0] for r in tables] + for i, table_name in enumerate(table_names): + pretty_name = NAME_MAPPING.get(table_name, table_name) + print("*" + pretty_name + "*") + # http://www.sqlite.org/faq.html#q24 + table_name = table_name.replace("\"", "\"\"") + rows = [] + for r in backend.engine.execute(SCHEMA_QUERY % table_name): + # Cut out the numbers from things like VARCHAR(12) since + # this is not very useful to show users who just want to + # see the basic schema... + row_type = re.sub(r"\(.*?\)", "", r['type']).strip() + if not row_type: + raise ValueError("Row %s of table '%s' was empty after" + " cleaning" % (r['cid'], table_name)) + rows.append([r['name'], row_type, to_bool_string(r['pk'])]) + contents = tabulate.tabulate( + rows, headers=['Name', 'Type', 'Primary Key'], + tablefmt="rst") + print("\n%s" % contents.strip()) + if i + 1 != len(table_names): + print("") + + +if __name__ == '__main__': + main() From 453323900bc7fb0d43e9465265951a30f34e28ca Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Sat, 11 Oct 2014 15:57:19 +0000 Subject: [PATCH 066/240] Updated from global requirements Change-Id: Ic0d84da1e3f06679be20973c8f618f561a76971d --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 19856bb6..20fc534c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -23,5 +23,5 @@ alembic>=0.6.4 psycopg2 # Docs build jobs need these packages. -sphinx>=1.1.2,!=1.2.0,<1.3 +sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 oslosphinx>=2.2.0 # Apache-2.0 From bcae66b21cdee23c8c857500a48d26f53d9ff6cb Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 14 Oct 2014 12:56:00 -0700 Subject: [PATCH 067/240] We can now use PyMySQL in py3.x tests The requirement for PyMySQL got merged into the requirements repo, so now instead of just testing sqlite (or postgres) in py3.x we can now test against the mysql instance that should exist in the test environments (just like we test with MySQL-python in py2.x). Change-Id: I6e48df78d20389d65de8210c6157abb99f52d7dd --- test-requirements.txt | 5 +++++ tox.ini | 2 ++ 2 files changed, 7 insertions(+) diff --git a/test-requirements.txt b/test-requirements.txt index 19856bb6..f2c68d26 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,6 +19,11 @@ kazoo>=1.3.1 # NOTE(harlowja): SQLAlchemy isn't listed here currently but is # listed in our tox.ini files so that we can test multiple varying SQLAlchemy # versions to ensure a wider range of compatibility. +# +# Explict mysql drivers are also not listed here so that we can test against +# PyMySQL or MySQL-python depending on the python version the tests are being +# ran in (MySQL-python is currently preferred for 2.x environments, since +# it has been used in openstack for the longest). alembic>=0.6.4 psycopg2 diff --git a/tox.ini b/tox.ini index 849820d4..99f11b23 100644 --- a/tox.ini +++ b/tox.ini @@ -84,11 +84,13 @@ commands = deps = {[testenv]deps} -r{toxinidir}/requirements-py3.txt SQLAlchemy>=0.7.8,<=0.9.99 + PyMySQL>=0.6.2 [testenv:py34] deps = {[testenv]deps} -r{toxinidir}/requirements-py3.txt SQLAlchemy>=0.7.8,<=0.9.99 + PyMySQL>=0.6.2 [testenv:py26-sa7-mysql] basepython = python2.6 From 8e6617716a6d8ba4dcb71fe886ef56620e699144 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 14 Oct 2014 13:37:32 -0700 Subject: [PATCH 068/240] Improve some of the task docstrings Change-Id: I72283d8043f66eb1de6af38b7fa17b261ab7fae8 --- taskflow/task.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/taskflow/task.py b/taskflow/task.py index cd470e72..6e5370e3 100644 --- a/taskflow/task.py +++ b/taskflow/task.py @@ -203,20 +203,21 @@ class BaseTask(atom.Atom): class Task(BaseTask): - """Base class for user-defined tasks. + """Base class for user-defined tasks (derive from it at will!). - Adds following features to Task: - - auto-generates name from type of self - - adds all execute argument names to task requirements - - items provided by the task may be specified via - 'default_provides' class attribute or property + Adds the following features on top of the :py:class:`.BaseTask`: + + - Auto-generates a name from the class name if a name is not + explicitly provided. + - Automatically adds all :py:meth:`.BaseTask.execute` argument names to + the task requirements (items provided by the task may be also specified + via ``default_provides`` class attribute or instance property). """ default_provides = None def __init__(self, name=None, provides=None, requires=None, auto_extract=True, rebind=None, inject=None): - """Initialize task instance.""" if provides is None: provides = self.default_provides super(Task, self).__init__(name, provides=provides, inject=inject) @@ -226,7 +227,11 @@ class Task(BaseTask): class FunctorTask(BaseTask): """Adaptor to make a task from a callable. - Take any callable and make a task from it. + Take any callable pair and make a task from it. + + NOTE(harlowja): If a name is not provided the function/method name of + the ``execute`` callable will be used as the name instead (the name of + the ``revert`` callable is not used). """ def __init__(self, execute, name=None, provides=None, From a15e07a0a12c808565f62c29007a61a1b0114809 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 16 Oct 2014 17:35:19 -0700 Subject: [PATCH 069/240] Use constants for revert automatically provided kwargs Instead of using strings use module level constants for the automatically provided keyword arguments to the tasks revert function. This makes it easier for users of taskflow to associate these constants with the actual keywords, without having to resort to using raw strings directly. Change-Id: I12726812615052d8405071d46272ef2b8286cfe2 --- taskflow/engines/action_engine/executor.py | 5 +++-- taskflow/task.py | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index b2bdbdae..40a671eb 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -19,6 +19,7 @@ import abc from concurrent import futures import six +from taskflow import task as _task from taskflow.utils import async_utils from taskflow.utils import misc from taskflow.utils import threading_utils @@ -44,8 +45,8 @@ def _execute_task(task, arguments, progress_callback): def _revert_task(task, arguments, result, failures, progress_callback): kwargs = arguments.copy() - kwargs['result'] = result - kwargs['flow_failures'] = failures + kwargs[_task.REVERT_RESULT] = result + kwargs[_task.REVERT_FLOW_FAILURES] = failures with task.autobind('update_progress', progress_callback): try: task.pre_revert() diff --git a/taskflow/task.py b/taskflow/task.py index cd470e72..211f09cc 100644 --- a/taskflow/task.py +++ b/taskflow/task.py @@ -27,6 +27,14 @@ from taskflow.utils import reflection LOG = logging.getLogger(__name__) +# Constants passed into revert kwargs. +# +# Contain the execute() result (if any). +REVERT_RESULT = 'result' +# +# The cause of the flow failure/s +REVERT_FLOW_FAILURES = 'flow_failures' + @six.add_metaclass(abc.ABCMeta) class BaseTask(atom.Atom): From f2ea4f128808316708a3854b3a7055e89773c061 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 23 Jun 2014 19:08:43 -0700 Subject: [PATCH 070/240] Move failure to its own type specific module The failure module needs to be hoisted out of the misc utility file so that it can be depended on existing by users in a well defined (non-utility) location. This change does this hoisting process & creates a new module and places the existing code there, then creates a deprecated proxy that exists at the old location (this will be removed in the next version + 1). In a future change (in 0.5) we can remove this old location and remove all references to the previous location (until then we must keep the old location being used to ensure subclass checks and other types checks function properly). Part of blueprint top-level-types Change-Id: I7d13ad1e9e5f5ecc90ab81949cc92ddf7309f13c --- doc/source/types.rst | 2 +- ...{test_utils_failure.py => test_failure.py} | 106 +++++-- taskflow/tests/unit/test_retries.py | 8 +- taskflow/tests/unit/test_utils.py | 47 +-- taskflow/types/failure.py | 287 ++++++++++++++++++ taskflow/utils/deprecation.py | 118 +++++++ taskflow/utils/misc.py | 271 +---------------- 7 files changed, 494 insertions(+), 345 deletions(-) rename taskflow/tests/unit/{test_utils_failure.py => test_failure.py} (73%) create mode 100644 taskflow/types/failure.py create mode 100644 taskflow/utils/deprecation.py diff --git a/doc/source/types.rst b/doc/source/types.rst index 1f573c8a..5c53db72 100644 --- a/doc/source/types.rst +++ b/doc/source/types.rst @@ -10,7 +10,7 @@ Cache Failure ======= -.. autoclass:: taskflow.utils.misc.Failure +.. automodule:: taskflow.types.failure FSM === diff --git a/taskflow/tests/unit/test_utils_failure.py b/taskflow/tests/unit/test_failure.py similarity index 73% rename from taskflow/tests/unit/test_utils_failure.py rename to taskflow/tests/unit/test_failure.py index 4958da62..3f4d001e 100644 --- a/taskflow/tests/unit/test_utils_failure.py +++ b/taskflow/tests/unit/test_failure.py @@ -14,19 +14,29 @@ # License for the specific language governing permissions and limitations # under the License. +import sys + import six from taskflow import exceptions from taskflow import test from taskflow.tests import utils as test_utils +from taskflow.types import failure from taskflow.utils import misc def _captured_failure(msg): - try: - raise RuntimeError(msg) - except Exception: - return misc.Failure() + try: + raise RuntimeError(msg) + except Exception: + return failure.Failure() + + +def _make_exc_info(msg): + try: + raise RuntimeError(msg) + except Exception: + return sys.exc_info() class GeneralFailureObjTestsMixin(object): @@ -85,9 +95,9 @@ class ReCreatedFailureTestCase(test.TestCase, GeneralFailureObjTestsMixin): def setUp(self): super(ReCreatedFailureTestCase, self).setUp() fail_obj = _captured_failure('Woot!') - self.fail_obj = misc.Failure(exception_str=fail_obj.exception_str, - traceback_str=fail_obj.traceback_str, - exc_type_names=list(fail_obj)) + self.fail_obj = failure.Failure(exception_str=fail_obj.exception_str, + traceback_str=fail_obj.traceback_str, + exc_type_names=list(fail_obj)) def test_value_lost(self): self.assertIs(self.fail_obj.exception, None) @@ -109,7 +119,7 @@ class FromExceptionTestCase(test.TestCase, GeneralFailureObjTestsMixin): def setUp(self): super(FromExceptionTestCase, self).setUp() - self.fail_obj = misc.Failure.from_exception(RuntimeError('Woot!')) + self.fail_obj = failure.Failure.from_exception(RuntimeError('Woot!')) def test_pformat_no_traceback(self): text = self.fail_obj.pformat(traceback=True) @@ -122,10 +132,10 @@ class FailureObjectTestCase(test.TestCase): try: raise SystemExit() except BaseException: - self.assertRaises(TypeError, misc.Failure) + self.assertRaises(TypeError, failure.Failure) def test_unknown_argument(self): - exc = self.assertRaises(TypeError, misc.Failure, + exc = self.assertRaises(TypeError, failure.Failure, exception_str='Woot!', traceback_str=None, exc_type_names=['Exception'], @@ -134,12 +144,12 @@ class FailureObjectTestCase(test.TestCase): self.assertEqual(str(exc), expected) def test_empty_does_not_reraise(self): - self.assertIs(misc.Failure.reraise_if_any([]), None) + self.assertIs(failure.Failure.reraise_if_any([]), None) def test_reraises_one(self): fls = [_captured_failure('Woot!')] self.assertRaisesRegexp(RuntimeError, '^Woot!$', - misc.Failure.reraise_if_any, fls) + failure.Failure.reraise_if_any, fls) def test_reraises_several(self): fls = [ @@ -147,7 +157,7 @@ class FailureObjectTestCase(test.TestCase): _captured_failure('Oh, not again!') ] exc = self.assertRaises(exceptions.WrappedFailure, - misc.Failure.reraise_if_any, fls) + failure.Failure.reraise_if_any, fls) self.assertEqual(list(exc), fls) def test_failure_copy(self): @@ -160,9 +170,9 @@ class FailureObjectTestCase(test.TestCase): def test_failure_copy_recaptured(self): captured = _captured_failure('Woot!') - fail_obj = misc.Failure(exception_str=captured.exception_str, - traceback_str=captured.traceback_str, - exc_type_names=list(captured)) + fail_obj = failure.Failure(exception_str=captured.exception_str, + traceback_str=captured.traceback_str, + exc_type_names=list(captured)) copied = fail_obj.copy() self.assertIsNot(fail_obj, copied) self.assertEqual(fail_obj, copied) @@ -171,9 +181,9 @@ class FailureObjectTestCase(test.TestCase): def test_recaptured_not_eq(self): captured = _captured_failure('Woot!') - fail_obj = misc.Failure(exception_str=captured.exception_str, - traceback_str=captured.traceback_str, - exc_type_names=list(captured)) + fail_obj = failure.Failure(exception_str=captured.exception_str, + traceback_str=captured.traceback_str, + exc_type_names=list(captured)) self.assertFalse(fail_obj == captured) self.assertTrue(fail_obj != captured) self.assertTrue(fail_obj.matches(captured)) @@ -185,13 +195,13 @@ class FailureObjectTestCase(test.TestCase): def test_two_recaptured_neq(self): captured = _captured_failure('Woot!') - fail_obj = misc.Failure(exception_str=captured.exception_str, - traceback_str=captured.traceback_str, - exc_type_names=list(captured)) + fail_obj = failure.Failure(exception_str=captured.exception_str, + traceback_str=captured.traceback_str, + exc_type_names=list(captured)) new_exc_str = captured.exception_str.replace('Woot', 'w00t') - fail_obj2 = misc.Failure(exception_str=new_exc_str, - traceback_str=captured.traceback_str, - exc_type_names=list(captured)) + fail_obj2 = failure.Failure(exception_str=new_exc_str, + traceback_str=captured.traceback_str, + exc_type_names=list(captured)) self.assertNotEqual(fail_obj, fail_obj2) self.assertFalse(fail_obj2.matches(fail_obj)) @@ -242,7 +252,7 @@ class WrappedFailureTestCase(test.TestCase): try: raise exceptions.WrappedFailure([f1, f2]) except Exception: - fail_obj = misc.Failure() + fail_obj = failure.Failure() wf = exceptions.WrappedFailure([fail_obj, f3]) self.assertEqual(list(wf), [f1, f2, f3]) @@ -252,13 +262,13 @@ class NonAsciiExceptionsTestCase(test.TestCase): def test_exception_with_non_ascii_str(self): bad_string = chr(200) - fail = misc.Failure.from_exception(ValueError(bad_string)) + fail = failure.Failure.from_exception(ValueError(bad_string)) self.assertEqual(fail.exception_str, bad_string) self.assertEqual(str(fail), 'Failure: ValueError: %s' % bad_string) def test_exception_non_ascii_unicode(self): hi_ru = u'привет' - fail = misc.Failure.from_exception(ValueError(hi_ru)) + fail = failure.Failure.from_exception(ValueError(hi_ru)) self.assertEqual(fail.exception_str, hi_ru) self.assertIsInstance(fail.exception_str, six.text_type) self.assertEqual(six.text_type(fail), @@ -268,7 +278,7 @@ class NonAsciiExceptionsTestCase(test.TestCase): hi_cn = u'嗨' fail = ValueError(hi_cn) self.assertEqual(hi_cn, exceptions.exception_message(fail)) - fail = misc.Failure.from_exception(fail) + fail = failure.Failure.from_exception(fail) wrapped_fail = exceptions.WrappedFailure([fail]) if six.PY2: # Python 2.x will unicode escape it, while python 3.3+ will not, @@ -283,12 +293,46 @@ class NonAsciiExceptionsTestCase(test.TestCase): def test_failure_equality_with_non_ascii_str(self): bad_string = chr(200) - fail = misc.Failure.from_exception(ValueError(bad_string)) + fail = failure.Failure.from_exception(ValueError(bad_string)) copied = fail.copy() self.assertEqual(fail, copied) def test_failure_equality_non_ascii_unicode(self): hi_ru = u'привет' - fail = misc.Failure.from_exception(ValueError(hi_ru)) + fail = failure.Failure.from_exception(ValueError(hi_ru)) copied = fail.copy() self.assertEqual(fail, copied) + + +class ExcInfoUtilsTest(test.TestCase): + def test_copy_none(self): + result = failure._copy_exc_info(None) + self.assertIsNone(result) + + def test_copy_exc_info(self): + exc_info = _make_exc_info("Woot!") + result = failure._copy_exc_info(exc_info) + self.assertIsNot(result, exc_info) + self.assertIs(result[0], RuntimeError) + self.assertIsNot(result[1], exc_info[1]) + self.assertIs(result[2], exc_info[2]) + + def test_none_equals(self): + self.assertTrue(failure._are_equal_exc_info_tuples(None, None)) + + def test_none_ne_tuple(self): + exc_info = _make_exc_info("Woot!") + self.assertFalse(failure._are_equal_exc_info_tuples(None, exc_info)) + + def test_tuple_nen_none(self): + exc_info = _make_exc_info("Woot!") + self.assertFalse(failure._are_equal_exc_info_tuples(exc_info, None)) + + def test_tuple_equals_itself(self): + exc_info = _make_exc_info("Woot!") + self.assertTrue(failure._are_equal_exc_info_tuples(exc_info, exc_info)) + + def test_typle_equals_copy(self): + exc_info = _make_exc_info("Woot!") + copied = failure._copy_exc_info(exc_info) + self.assertTrue(failure._are_equal_exc_info_tuples(exc_info, copied)) diff --git a/taskflow/tests/unit/test_retries.py b/taskflow/tests/unit/test_retries.py index 71ea70cb..fb1b3802 100644 --- a/taskflow/tests/unit/test_retries.py +++ b/taskflow/tests/unit/test_retries.py @@ -23,7 +23,7 @@ from taskflow import retry from taskflow import states as st from taskflow import test from taskflow.tests import utils -from taskflow.utils import misc +from taskflow.types import failure class RetryTest(utils.EngineTestBase): @@ -559,7 +559,7 @@ class RetryTest(utils.EngineTestBase): # we execute retry engine.storage.save('flow-1_retry', 1) # task fails - fail = misc.Failure.from_exception(RuntimeError('foo')), + fail = failure.Failure.from_exception(RuntimeError('foo')), engine.storage.save('task1', fail, state=st.FAILURE) if when == 'task fails': return engine @@ -635,7 +635,7 @@ class RetryTest(utils.EngineTestBase): self._make_engine(flow).run) self.assertEqual(len(r.history), 1) self.assertEqual(r.history[0][1], {}) - self.assertEqual(isinstance(r.history[0][0], misc.Failure), True) + self.assertEqual(isinstance(r.history[0][0], failure.Failure), True) def test_retry_revert_fails(self): @@ -693,7 +693,7 @@ class RetryTest(utils.EngineTestBase): engine.storage.save('test2_retry', 1) engine.storage.save('b', 11) # pretend that 'c' failed - fail = misc.Failure.from_exception(RuntimeError('Woot!')) + fail = failure.Failure.from_exception(RuntimeError('Woot!')) engine.storage.save('c', fail, st.FAILURE) engine.run() diff --git a/taskflow/tests/unit/test_utils.py b/taskflow/tests/unit/test_utils.py index 5d4b7615..c9c93f3c 100644 --- a/taskflow/tests/unit/test_utils.py +++ b/taskflow/tests/unit/test_utils.py @@ -18,7 +18,6 @@ import collections import functools import inspect import random -import sys import threading import time @@ -28,6 +27,7 @@ import testtools from taskflow import states from taskflow import test from taskflow.tests import utils as test_utils +from taskflow.types import failure from taskflow.utils import lock_utils from taskflow.utils import misc from taskflow.utils import reflection @@ -335,8 +335,8 @@ class GetClassNameTest(test.TestCase): self.assertEqual(name, 'RuntimeError') def test_global_class(self): - name = reflection.get_class_name(misc.Failure) - self.assertEqual(name, 'taskflow.utils.misc.Failure') + name = reflection.get_class_name(failure.Failure) + self.assertEqual(name, 'taskflow.types.failure.Failure') def test_class(self): name = reflection.get_class_name(Class) @@ -619,47 +619,6 @@ class UriParseTest(test.TestCase): self.assertEqual(None, parsed.password) -class ExcInfoUtilsTest(test.TestCase): - - def _make_ex_info(self): - try: - raise RuntimeError('Woot!') - except Exception: - return sys.exc_info() - - def test_copy_none(self): - result = misc.copy_exc_info(None) - self.assertIsNone(result) - - def test_copy_exc_info(self): - exc_info = self._make_ex_info() - result = misc.copy_exc_info(exc_info) - self.assertIsNot(result, exc_info) - self.assertIs(result[0], RuntimeError) - self.assertIsNot(result[1], exc_info[1]) - self.assertIs(result[2], exc_info[2]) - - def test_none_equals(self): - self.assertTrue(misc.are_equal_exc_info_tuples(None, None)) - - def test_none_ne_tuple(self): - exc_info = self._make_ex_info() - self.assertFalse(misc.are_equal_exc_info_tuples(None, exc_info)) - - def test_tuple_nen_none(self): - exc_info = self._make_ex_info() - self.assertFalse(misc.are_equal_exc_info_tuples(exc_info, None)) - - def test_tuple_equals_itself(self): - exc_info = self._make_ex_info() - self.assertTrue(misc.are_equal_exc_info_tuples(exc_info, exc_info)) - - def test_typle_equals_copy(self): - exc_info = self._make_ex_info() - copied = misc.copy_exc_info(exc_info) - self.assertTrue(misc.are_equal_exc_info_tuples(exc_info, copied)) - - class TestSequenceMinus(test.TestCase): def test_simple_case(self): diff --git a/taskflow/types/failure.py b/taskflow/types/failure.py new file mode 100644 index 00000000..93829ad6 --- /dev/null +++ b/taskflow/types/failure.py @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import sys +import traceback + +import six + +from taskflow import exceptions as exc +from taskflow.utils import reflection + + +def _copy_exc_info(exc_info): + """Make copy of exception info tuple, as deep as possible.""" + if exc_info is None: + return None + exc_type, exc_value, tb = exc_info + # NOTE(imelnikov): there is no need to copy type, and + # we can't copy traceback. + return (exc_type, copy.deepcopy(exc_value), tb) + + +def _are_equal_exc_info_tuples(ei1, ei2): + if ei1 == ei2: + return True + if ei1 is None or ei2 is None: + return False # if both are None, we returned True above + + # NOTE(imelnikov): we can't compare exceptions with '==' + # because we want exc_info be equal to it's copy made with + # copy_exc_info above. + if ei1[0] is not ei2[0]: + return False + if not all((type(ei1[1]) == type(ei2[1]), + exc.exception_message(ei1[1]) == exc.exception_message(ei2[1]), + repr(ei1[1]) == repr(ei2[1]))): + return False + if ei1[2] == ei2[2]: + return True + tb1 = traceback.format_tb(ei1[2]) + tb2 = traceback.format_tb(ei2[2]) + return tb1 == tb2 + + +class Failure(object): + """Object that represents failure. + + Failure objects encapsulate exception information so that they can be + re-used later to re-raise, inspect, examine, log, print, serialize, + deserialize... + + One example where they are dependened upon is in the WBE engine. When a + remote worker throws an exception, the WBE based engine will receive that + exception and desire to reraise it to the user/caller of the WBE based + engine for appropriate handling (this matches the behavior of non-remote + engines). To accomplish this a failure object (or a + :py:meth:`~misc.Failure.to_dict` form) would be sent over the WBE channel + and the WBE based engine would deserialize it and use this objects + :meth:`.reraise` method to cause an exception that contains + similar/equivalent information as the original exception to be reraised, + allowing the user (or the WBE engine itself) to then handle the worker + failure/exception as they desire. + + For those who are curious, here are a few reasons why the original + exception itself *may* not be reraised and instead a reraised wrapped + failure exception object will be instead. These explanations are *only* + applicable when a failure object is serialized and deserialized (when it is + retained inside the python process that the exception was created in the + the original exception can be reraised correctly without issue). + + * Traceback objects are not serializable/recreatable, since they contain + references to stack frames at the location where the exception was + raised. When a failure object is serialized and sent across a channel + and recreated it is *not* possible to restore the original traceback and + originating stack frames. + * The original exception *type* can not be guaranteed to be found, workers + can run code that is not accessible/available when the failure is being + deserialized. Even if it was possible to use pickle safely it would not + be possible to find the originating exception or associated code in this + situation. + * The original exception *type* can not be guaranteed to be constructed in + a *correct* manner. At the time of failure object creation the exception + has already been created and the failure object can not assume it has + knowledge (or the ability) to recreate the original type of the captured + exception (this is especially hard if the original exception was created + via a complex process via some custom exception constructor). + * The original exception *type* can not be guaranteed to be constructed in + a *safe* manner. Importing *foreign* exception types dynamically can be + problematic when not done correctly and in a safe manner; since failure + objects can capture any exception it would be *unsafe* to try to import + those exception types namespaces and modules on the receiver side + dynamically (this would create similar issues as the ``pickle`` module in + python has where foreign modules can be imported, causing those modules + to have code ran when this happens, and this can cause issues and + side-effects that the receiver would not have intended to have caused). + """ + DICT_VERSION = 1 + + def __init__(self, exc_info=None, **kwargs): + if not kwargs: + if exc_info is None: + exc_info = sys.exc_info() + self._exc_info = exc_info + self._exc_type_names = list( + reflection.get_all_class_names(exc_info[0], up_to=Exception)) + if not self._exc_type_names: + raise TypeError('Invalid exception type: %r' % exc_info[0]) + self._exception_str = exc.exception_message(self._exc_info[1]) + self._traceback_str = ''.join( + traceback.format_tb(self._exc_info[2])) + else: + self._exc_info = exc_info # may be None + self._exception_str = kwargs.pop('exception_str') + self._exc_type_names = kwargs.pop('exc_type_names', []) + self._traceback_str = kwargs.pop('traceback_str', None) + if kwargs: + raise TypeError( + 'Failure.__init__ got unexpected keyword argument(s): %s' + % ', '.join(six.iterkeys(kwargs))) + + @classmethod + def from_exception(cls, exception): + """Creates a failure object from a exception instance.""" + return cls((type(exception), exception, None)) + + def _matches(self, other): + if self is other: + return True + return (self._exc_type_names == other._exc_type_names + and self.exception_str == other.exception_str + and self.traceback_str == other.traceback_str) + + def matches(self, other): + """Checks if another object is equivalent to this object.""" + if not isinstance(other, Failure): + return False + if self.exc_info is None or other.exc_info is None: + return self._matches(other) + else: + return self == other + + def __eq__(self, other): + if not isinstance(other, Failure): + return NotImplemented + return (self._matches(other) and + _are_equal_exc_info_tuples(self.exc_info, other.exc_info)) + + def __ne__(self, other): + return not (self == other) + + # NOTE(imelnikov): obj.__hash__() should return same values for equal + # objects, so we should redefine __hash__. Failure equality semantics + # is a bit complicated, so for now we just mark Failure objects as + # unhashable. See python docs on object.__hash__ for more info: + # http://docs.python.org/2/reference/datamodel.html#object.__hash__ + __hash__ = None + + @property + def exception(self): + """Exception value, or None if exception value is not present. + + Exception value may be lost during serialization. + """ + if self._exc_info: + return self._exc_info[1] + else: + return None + + @property + def exception_str(self): + """String representation of exception.""" + return self._exception_str + + @property + def exc_info(self): + """Exception info tuple or None.""" + return self._exc_info + + @property + def traceback_str(self): + """Exception traceback as string.""" + return self._traceback_str + + @staticmethod + def reraise_if_any(failures): + """Re-raise exceptions if argument is not empty. + + If argument is empty list, this method returns None. If + argument is a list with a single ``Failure`` object in it, + that failure is reraised. Else, a + :class:`~taskflow.exceptions.WrappedFailure` exception + is raised with a failure list as causes. + """ + failures = list(failures) + if len(failures) == 1: + failures[0].reraise() + elif len(failures) > 1: + raise exc.WrappedFailure(failures) + + def reraise(self): + """Re-raise captured exception.""" + if self._exc_info: + six.reraise(*self._exc_info) + else: + raise exc.WrappedFailure([self]) + + def check(self, *exc_classes): + """Check if any of ``exc_classes`` caused the failure. + + Arguments of this method can be exception types or type + names (stings). If captured exception is instance of + exception of given type, the corresponding argument is + returned. Else, None is returned. + """ + for cls in exc_classes: + if isinstance(cls, type): + err = reflection.get_class_name(cls) + else: + err = cls + if err in self._exc_type_names: + return cls + return None + + def __str__(self): + return self.pformat() + + def pformat(self, traceback=False): + """Pretty formats the failure object into a string.""" + buf = six.StringIO() + buf.write( + 'Failure: %s: %s' % (self._exc_type_names[0], self._exception_str)) + if traceback: + if self._traceback_str is not None: + traceback_str = self._traceback_str.rstrip() + else: + traceback_str = None + if traceback_str: + buf.write('\nTraceback (most recent call last):\n') + buf.write(traceback_str) + else: + buf.write('\nTraceback not available.') + return buf.getvalue() + + def __iter__(self): + """Iterate over exception type names.""" + for et in self._exc_type_names: + yield et + + @classmethod + def from_dict(cls, data): + """Converts this from a dictionary to a object.""" + data = dict(data) + version = data.pop('version', None) + if version != cls.DICT_VERSION: + raise ValueError('Invalid dict version of failure object: %r' + % version) + return cls(**data) + + def to_dict(self): + """Converts this object to a dictionary.""" + return { + 'exception_str': self.exception_str, + 'traceback_str': self.traceback_str, + 'exc_type_names': list(self), + 'version': self.DICT_VERSION, + } + + def copy(self): + """Copies this object.""" + return Failure(exc_info=_copy_exc_info(self.exc_info), + exception_str=self.exception_str, + traceback_str=self.traceback_str, + exc_type_names=self._exc_type_names[:]) diff --git a/taskflow/utils/deprecation.py b/taskflow/utils/deprecation.py new file mode 100644 index 00000000..899e035d --- /dev/null +++ b/taskflow/utils/deprecation.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import warnings + +from taskflow.utils import reflection + + +def deprecation(message, stacklevel=2): + """Warns about some type of deprecation that has been made.""" + warnings.warn(message, category=DeprecationWarning, stacklevel=stacklevel) + + +# Helper accessors for the moved proxy (since it will not have easy access +# to its own getattr and setattr functions). +_setattr = object.__setattr__ +_getattr = object.__getattribute__ + + +class MovedClassProxy(object): + """Acts as a proxy to a class that was moved to another location. + + Partially based on: + + http://code.activestate.com/recipes/496741-object-proxying/ and other + various examination of how to make a good enough proxy for our usage to + move the various types we want to move during the deprecation process. + + And partially based on the wrapt object proxy (which we should just use + when it becomes available @ http://review.openstack.org/#/c/94754/). + """ + + __slots__ = [ + '__wrapped__', '__message__', '__stacklevel__', + # Ensure weakrefs can be made, + # https://docs.python.org/2/reference/datamodel.html#slots + '__weakref__', + ] + + def __init__(self, wrapped, message, stacklevel): + # We can't assign to these directly, since we are overriding getattr + # and setattr and delattr so we have to do this hoop jump to ensure + # that we don't invoke those methods (and cause infinite recursion). + _setattr(self, '__wrapped__', wrapped) + _setattr(self, '__message__', message) + _setattr(self, '__stacklevel__', stacklevel) + try: + _setattr(self, '__qualname__', wrapped.__qualname__) + except AttributeError: + pass + + def __instancecheck__(self, instance): + deprecation( + _getattr(self, '__message__'), _getattr(self, '__stacklevel__')) + return isinstance(instance, _getattr(self, '__wrapped__')) + + def __subclasscheck__(self, instance): + deprecation( + _getattr(self, '__message__'), _getattr(self, '__stacklevel__')) + return issubclass(instance, _getattr(self, '__wrapped__')) + + def __call__(self, *args, **kwargs): + deprecation( + _getattr(self, '__message__'), _getattr(self, '__stacklevel__')) + return _getattr(self, '__wrapped__')(*args, **kwargs) + + def __getattribute__(self, name): + return getattr(_getattr(self, '__wrapped__'), name) + + def __setattr__(self, name, value): + setattr(_getattr(self, '__wrapped__'), name, value) + + def __delattr__(self, name): + delattr(_getattr(self, '__wrapped__'), name) + + def __repr__(self): + wrapped = _getattr(self, '__wrapped__') + return "<%s at 0x%x for %r at 0x%x>" % ( + type(self).__name__, id(self), wrapped, id(wrapped)) + + +def moved_class(new_class, old_class_name, old_module_name, message=None, + version=None, removal_version=None): + """Deprecates a class that was moved to another location. + + This will emit warnings when the old locations class is initialized, + telling where the new and improved location for the old class now is. + """ + old_name = ".".join((old_module_name, old_class_name)) + new_name = reflection.get_class_name(new_class) + message_components = [ + "Class '%s' has moved to '%s'" % (old_name, new_name), + ] + if version: + message_components.append(" in version '%s'" % version) + if removal_version: + if removal_version == "?": + message_components.append(" and will be removed in a future" + " version") + else: + message_components.append(" and will be removed in version '%s'" + % removal_version) + if message: + message_components.append(": %s" % message) + return MovedClassProxy(new_class, "".join(message_components), 3) diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 147085dd..41c5cfcd 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -28,7 +28,6 @@ import re import string import sys import threading -import traceback from oslo.serialization import jsonutils from oslo.utils import netutils @@ -37,7 +36,8 @@ from six.moves import map as compat_map from six.moves import range as compat_range from six.moves.urllib import parse as urlparse -from taskflow import exceptions as exc +from taskflow.types import failure +from taskflow.utils import deprecation from taskflow.utils import reflection @@ -392,6 +392,10 @@ def ensure_tree(path): raise +Failure = deprecation.moved_class(failure.Failure, 'Failure', __name__, + version="0.5", removal_version="?") + + class Notifier(object): """A notification helper class. @@ -489,38 +493,6 @@ class Notifier(object): break -def copy_exc_info(exc_info): - """Make copy of exception info tuple, as deep as possible.""" - if exc_info is None: - return None - exc_type, exc_value, tb = exc_info - # NOTE(imelnikov): there is no need to copy type, and - # we can't copy traceback. - return (exc_type, copy.deepcopy(exc_value), tb) - - -def are_equal_exc_info_tuples(ei1, ei2): - if ei1 == ei2: - return True - if ei1 is None or ei2 is None: - return False # if both are None, we returned True above - - # NOTE(imelnikov): we can't compare exceptions with '==' - # because we want exc_info be equal to it's copy made with - # copy_exc_info above. - if ei1[0] is not ei2[0]: - return False - if not all((type(ei1[1]) == type(ei2[1]), - exc.exception_message(ei1[1]) == exc.exception_message(ei2[1]), - repr(ei1[1]) == repr(ei2[1]))): - return False - if ei1[2] == ei2[2]: - return True - tb1 = traceback.format_tb(ei1[2]) - tb2 = traceback.format_tb(ei2[2]) - return tb1 == tb2 - - @contextlib.contextmanager def capture_failure(): """Captures the occurring exception and provides a failure object back. @@ -551,234 +523,3 @@ def capture_failure(): raise RuntimeError("No active exception is being handled") else: yield Failure(exc_info=exc_info) - - -class Failure(object): - """Object that represents failure. - - Failure objects encapsulate exception information so that they can be - re-used later to re-raise, inspect, examine, log, print, serialize, - deserialize... - - One example where they are dependened upon is in the WBE engine. When a - remote worker throws an exception, the WBE based engine will receive that - exception and desire to reraise it to the user/caller of the WBE based - engine for appropriate handling (this matches the behavior of non-remote - engines). To accomplish this a failure object (or a - :py:meth:`~misc.Failure.to_dict` form) would be sent over the WBE channel - and the WBE based engine would deserialize it and use this objects - :meth:`.reraise` method to cause an exception that contains - similar/equivalent information as the original exception to be reraised, - allowing the user (or the WBE engine itself) to then handle the worker - failure/exception as they desire. - - For those who are curious, here are a few reasons why the original - exception itself *may* not be reraised and instead a reraised wrapped - failure exception object will be instead. These explanations are *only* - applicable when a failure object is serialized and deserialized (when it is - retained inside the python process that the exception was created in the - the original exception can be reraised correctly without issue). - - * Traceback objects are not serializable/recreatable, since they contain - references to stack frames at the location where the exception was - raised. When a failure object is serialized and sent across a channel - and recreated it is *not* possible to restore the original traceback and - originating stack frames. - * The original exception *type* can not be guaranteed to be found, workers - can run code that is not accessible/available when the failure is being - deserialized. Even if it was possible to use pickle safely it would not - be possible to find the originating exception or associated code in this - situation. - * The original exception *type* can not be guaranteed to be constructed in - a *correct* manner. At the time of failure object creation the exception - has already been created and the failure object can not assume it has - knowledge (or the ability) to recreate the original type of the captured - exception (this is especially hard if the original exception was created - via a complex process via some custom exception constructor). - * The original exception *type* can not be guaranteed to be constructed in - a *safe* manner. Importing *foreign* exception types dynamically can be - problematic when not done correctly and in a safe manner; since failure - objects can capture any exception it would be *unsafe* to try to import - those exception types namespaces and modules on the receiver side - dynamically (this would create similar issues as the ``pickle`` module in - python has where foreign modules can be imported, causing those modules - to have code ran when this happens, and this can cause issues and - side-effects that the receiver would not have intended to have caused). - """ - DICT_VERSION = 1 - - def __init__(self, exc_info=None, **kwargs): - if not kwargs: - if exc_info is None: - exc_info = sys.exc_info() - self._exc_info = exc_info - self._exc_type_names = list( - reflection.get_all_class_names(exc_info[0], up_to=Exception)) - if not self._exc_type_names: - raise TypeError('Invalid exception type: %r' % exc_info[0]) - self._exception_str = exc.exception_message(self._exc_info[1]) - self._traceback_str = ''.join( - traceback.format_tb(self._exc_info[2])) - else: - self._exc_info = exc_info # may be None - self._exception_str = kwargs.pop('exception_str') - self._exc_type_names = kwargs.pop('exc_type_names', []) - self._traceback_str = kwargs.pop('traceback_str', None) - if kwargs: - raise TypeError( - 'Failure.__init__ got unexpected keyword argument(s): %s' - % ', '.join(six.iterkeys(kwargs))) - - @classmethod - def from_exception(cls, exception): - """Creates a failure object from a exception instance.""" - return cls((type(exception), exception, None)) - - def _matches(self, other): - if self is other: - return True - return (self._exc_type_names == other._exc_type_names - and self.exception_str == other.exception_str - and self.traceback_str == other.traceback_str) - - def matches(self, other): - """Checks if another object is equivalent to this object.""" - if not isinstance(other, Failure): - return False - if self.exc_info is None or other.exc_info is None: - return self._matches(other) - else: - return self == other - - def __eq__(self, other): - if not isinstance(other, Failure): - return NotImplemented - return (self._matches(other) and - are_equal_exc_info_tuples(self.exc_info, other.exc_info)) - - def __ne__(self, other): - return not (self == other) - - # NOTE(imelnikov): obj.__hash__() should return same values for equal - # objects, so we should redefine __hash__. Failure equality semantics - # is a bit complicated, so for now we just mark Failure objects as - # unhashable. See python docs on object.__hash__ for more info: - # http://docs.python.org/2/reference/datamodel.html#object.__hash__ - __hash__ = None - - @property - def exception(self): - """Exception value, or None if exception value is not present. - - Exception value may be lost during serialization. - """ - if self._exc_info: - return self._exc_info[1] - else: - return None - - @property - def exception_str(self): - """String representation of exception.""" - return self._exception_str - - @property - def exc_info(self): - """Exception info tuple or None.""" - return self._exc_info - - @property - def traceback_str(self): - """Exception traceback as string.""" - return self._traceback_str - - @staticmethod - def reraise_if_any(failures): - """Re-raise exceptions if argument is not empty. - - If argument is empty list, this method returns None. If - argument is a list with a single ``Failure`` object in it, - that failure is reraised. Else, a - :class:`~taskflow.exceptions.WrappedFailure` exception - is raised with a failure list as causes. - """ - failures = list(failures) - if len(failures) == 1: - failures[0].reraise() - elif len(failures) > 1: - raise exc.WrappedFailure(failures) - - def reraise(self): - """Re-raise captured exception.""" - if self._exc_info: - six.reraise(*self._exc_info) - else: - raise exc.WrappedFailure([self]) - - def check(self, *exc_classes): - """Check if any of ``exc_classes`` caused the failure. - - Arguments of this method can be exception types or type - names (stings). If captured exception is instance of - exception of given type, the corresponding argument is - returned. Else, None is returned. - """ - for cls in exc_classes: - if isinstance(cls, type): - err = reflection.get_class_name(cls) - else: - err = cls - if err in self._exc_type_names: - return cls - return None - - def __str__(self): - return self.pformat() - - def pformat(self, traceback=False): - """Pretty formats the failure object into a string.""" - buf = six.StringIO() - buf.write( - 'Failure: %s: %s' % (self._exc_type_names[0], self._exception_str)) - if traceback: - if self._traceback_str is not None: - traceback_str = self._traceback_str.rstrip() - else: - traceback_str = None - if traceback_str: - buf.write('\nTraceback (most recent call last):\n') - buf.write(traceback_str) - else: - buf.write('\nTraceback not available.') - return buf.getvalue() - - def __iter__(self): - """Iterate over exception type names.""" - for et in self._exc_type_names: - yield et - - @classmethod - def from_dict(cls, data): - """Converts this from a dictionary to a object.""" - data = dict(data) - version = data.pop('version', None) - if version != cls.DICT_VERSION: - raise ValueError('Invalid dict version of failure object: %r' - % version) - return cls(**data) - - def to_dict(self): - """Converts this object to a dictionary.""" - return { - 'exception_str': self.exception_str, - 'traceback_str': self.traceback_str, - 'exc_type_names': list(self), - 'version': self.DICT_VERSION, - } - - def copy(self): - """Copies this object.""" - return Failure(exc_info=copy_exc_info(self.exc_info), - exception_str=self.exception_str, - traceback_str=self.traceback_str, - exc_type_names=self._exc_type_names[:]) From bf84288aa0fca3bc5b5582375d145d2966e10c0a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 24 Jun 2014 12:47:05 -0700 Subject: [PATCH 071/240] Hoist the notifier to its own module The notifier module needs to be hoisted out of the misc utility file so that it can be depended on existing by users in a well defined (non-utility) location. This change does this hoisting process & creates a new module and places the existing code there, then creates a deprecated proxy that exists at the old location (this will be removed in the next version + 1). In a future change (in 0.5) we can remove this old location and remove all references to the previous location (until then we must keep the old location being used to ensure subclass checks and other types checks function properly). Part of blueprint top-level-types Change-Id: I47fac110adf7cec5c859c2e055c1ceb1f25a7fbd --- doc/source/types.rst | 2 +- taskflow/tests/unit/test_notifier.py | 104 +++++++++++++++++++++++ taskflow/tests/unit/test_utils.py | 84 ------------------ taskflow/types/notifier.py | 122 +++++++++++++++++++++++++++ taskflow/utils/misc.py | 100 +--------------------- 5 files changed, 230 insertions(+), 182 deletions(-) create mode 100644 taskflow/tests/unit/test_notifier.py create mode 100644 taskflow/types/notifier.py diff --git a/doc/source/types.rst b/doc/source/types.rst index 5c53db72..c628c1ce 100644 --- a/doc/source/types.rst +++ b/doc/source/types.rst @@ -25,7 +25,7 @@ Graph Notifier ======== -.. autoclass:: taskflow.utils.misc.Notifier +.. automodule:: taskflow.types.notifier Table ===== diff --git a/taskflow/tests/unit/test_notifier.py b/taskflow/tests/unit/test_notifier.py new file mode 100644 index 00000000..0761cb6e --- /dev/null +++ b/taskflow/tests/unit/test_notifier.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2013 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +import functools + +from taskflow import states +from taskflow import test +from taskflow.types import notifier as nt + + +class NotifierTest(test.TestCase): + + def test_notify_called(self): + call_collector = [] + + def call_me(state, details): + call_collector.append((state, details)) + + notifier = nt.Notifier() + notifier.register(nt.Notifier.ANY, call_me) + notifier.notify(states.SUCCESS, {}) + notifier.notify(states.SUCCESS, {}) + + self.assertEqual(2, len(call_collector)) + self.assertEqual(1, len(notifier)) + + def test_notify_register_deregister(self): + + def call_me(state, details): + pass + + class A(object): + def call_me_too(self, state, details): + pass + + notifier = nt.Notifier() + notifier.register(nt.Notifier.ANY, call_me) + a = A() + notifier.register(nt.Notifier.ANY, a.call_me_too) + + self.assertEqual(2, len(notifier)) + notifier.deregister(nt.Notifier.ANY, call_me) + notifier.deregister(nt.Notifier.ANY, a.call_me_too) + self.assertEqual(0, len(notifier)) + + def test_notify_reset(self): + + def call_me(state, details): + pass + + notifier = nt.Notifier() + notifier.register(nt.Notifier.ANY, call_me) + self.assertEqual(1, len(notifier)) + + notifier.reset() + self.assertEqual(0, len(notifier)) + + def test_bad_notify(self): + + def call_me(state, details): + pass + + notifier = nt.Notifier() + self.assertRaises(KeyError, notifier.register, + nt.Notifier.ANY, call_me, + kwargs={'details': 5}) + + def test_selective_notify(self): + call_counts = collections.defaultdict(list) + + def call_me_on(registered_state, state, details): + call_counts[registered_state].append((state, details)) + + notifier = nt.Notifier() + notifier.register(states.SUCCESS, + functools.partial(call_me_on, states.SUCCESS)) + notifier.register(nt.Notifier.ANY, + functools.partial(call_me_on, + nt.Notifier.ANY)) + + self.assertEqual(2, len(notifier)) + notifier.notify(states.SUCCESS, {}) + + self.assertEqual(1, len(call_counts[nt.Notifier.ANY])) + self.assertEqual(1, len(call_counts[states.SUCCESS])) + + notifier.notify(states.FAILURE, {}) + self.assertEqual(2, len(call_counts[nt.Notifier.ANY])) + self.assertEqual(1, len(call_counts[states.SUCCESS])) + self.assertEqual(2, len(call_counts)) diff --git a/taskflow/tests/unit/test_utils.py b/taskflow/tests/unit/test_utils.py index c9c93f3c..e8660bd3 100644 --- a/taskflow/tests/unit/test_utils.py +++ b/taskflow/tests/unit/test_utils.py @@ -15,7 +15,6 @@ # under the License. import collections -import functools import inspect import random import threading @@ -24,7 +23,6 @@ import time import six import testtools -from taskflow import states from taskflow import test from taskflow.tests import utils as test_utils from taskflow.types import failure @@ -192,88 +190,6 @@ class GetCallableNameTestExtended(test.TestCase): self.assertEqual(expected_name, name) -class NotifierTest(test.TestCase): - - def test_notify_called(self): - call_collector = [] - - def call_me(state, details): - call_collector.append((state, details)) - - notifier = misc.Notifier() - notifier.register(misc.Notifier.ANY, call_me) - notifier.notify(states.SUCCESS, {}) - notifier.notify(states.SUCCESS, {}) - - self.assertEqual(2, len(call_collector)) - self.assertEqual(1, len(notifier)) - - def test_notify_register_deregister(self): - - def call_me(state, details): - pass - - class A(object): - def call_me_too(self, state, details): - pass - - notifier = misc.Notifier() - notifier.register(misc.Notifier.ANY, call_me) - a = A() - notifier.register(misc.Notifier.ANY, a.call_me_too) - - self.assertEqual(2, len(notifier)) - notifier.deregister(misc.Notifier.ANY, call_me) - notifier.deregister(misc.Notifier.ANY, a.call_me_too) - self.assertEqual(0, len(notifier)) - - def test_notify_reset(self): - - def call_me(state, details): - pass - - notifier = misc.Notifier() - notifier.register(misc.Notifier.ANY, call_me) - self.assertEqual(1, len(notifier)) - - notifier.reset() - self.assertEqual(0, len(notifier)) - - def test_bad_notify(self): - - def call_me(state, details): - pass - - notifier = misc.Notifier() - self.assertRaises(KeyError, notifier.register, - misc.Notifier.ANY, call_me, - kwargs={'details': 5}) - - def test_selective_notify(self): - call_counts = collections.defaultdict(list) - - def call_me_on(registered_state, state, details): - call_counts[registered_state].append((state, details)) - - notifier = misc.Notifier() - notifier.register(states.SUCCESS, - functools.partial(call_me_on, states.SUCCESS)) - notifier.register(misc.Notifier.ANY, - functools.partial(call_me_on, - misc.Notifier.ANY)) - - self.assertEqual(2, len(notifier)) - notifier.notify(states.SUCCESS, {}) - - self.assertEqual(1, len(call_counts[misc.Notifier.ANY])) - self.assertEqual(1, len(call_counts[states.SUCCESS])) - - notifier.notify(states.FAILURE, {}) - self.assertEqual(2, len(call_counts[misc.Notifier.ANY])) - self.assertEqual(1, len(call_counts[states.SUCCESS])) - self.assertEqual(2, len(call_counts)) - - class GetCallableArgsTest(test.TestCase): def test_mere_function(self): diff --git a/taskflow/types/notifier.py b/taskflow/types/notifier.py new file mode 100644 index 00000000..a92d6b89 --- /dev/null +++ b/taskflow/types/notifier.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +import copy +import logging + +import six + +from taskflow.utils import reflection + +LOG = logging.getLogger(__name__) + + +class Notifier(object): + """A notification helper class. + + It is intended to be used to subscribe to notifications of events + occurring as well as allow a entity to post said notifications to any + associated subscribers without having either entity care about how this + notification occurs. + """ + + #: Keys that can not be used in callbacks arguments + RESERVED_KEYS = ('details',) + + #: Kleene star constant that is used to recieve all notifications + ANY = '*' + + def __init__(self): + self._listeners = collections.defaultdict(list) + + def __len__(self): + """Returns how many callbacks are registered.""" + count = 0 + for (_event_type, callbacks) in six.iteritems(self._listeners): + count += len(callbacks) + return count + + def is_registered(self, event_type, callback): + """Check if a callback is registered.""" + listeners = list(self._listeners.get(event_type, [])) + for (cb, _args, _kwargs) in listeners: + if reflection.is_same_callback(cb, callback): + return True + return False + + def reset(self): + """Forget all previously registered callbacks.""" + self._listeners.clear() + + def notify(self, event_type, details): + """Notify about event occurrence. + + All callbacks registered to receive notifications about given + event type will be called. + + :param event_type: event type that occurred + :param details: addition event details + """ + listeners = list(self._listeners.get(self.ANY, [])) + for i in self._listeners[event_type]: + if i not in listeners: + listeners.append(i) + if not listeners: + return + for (callback, args, kwargs) in listeners: + if args is None: + args = [] + if kwargs is None: + kwargs = {} + kwargs['details'] = details + try: + callback(event_type, *args, **kwargs) + except Exception: + LOG.warn("Failure calling callback %s to notify about event" + " %s, details: %s", callback, event_type, + details, exc_info=True) + + def register(self, event_type, callback, args=None, kwargs=None): + """Register a callback to be called when event of a given type occurs. + + Callback will be called with provided ``args`` and ``kwargs`` and + when event type occurs (or on any event if ``event_type`` equals to + :attr:`.ANY`). It will also get additional keyword argument, + ``details``, that will hold event details provided to the + :meth:`.notify` method. + """ + assert six.callable(callback), "Callback must be callable" + if self.is_registered(event_type, callback): + raise ValueError("Callback %s already registered" % (callback)) + if kwargs: + for k in self.RESERVED_KEYS: + if k in kwargs: + raise KeyError(("Reserved key '%s' not allowed in " + "kwargs") % k) + kwargs = copy.copy(kwargs) + if args: + args = copy.copy(args) + self._listeners[event_type].append((callback, args, kwargs)) + + def deregister(self, event_type, callback): + """Remove a single callback from listening to event ``event_type``.""" + if event_type not in self._listeners: + return + for i, (cb, args, kwargs) in enumerate(self._listeners[event_type]): + if reflection.is_same_callback(cb, callback): + self._listeners[event_type].pop(i) + break diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 41c5cfcd..a4619881 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -15,9 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. -import collections import contextlib -import copy import datetime import errno import inspect @@ -37,6 +35,7 @@ from six.moves import range as compat_range from six.moves.urllib import parse as urlparse from taskflow.types import failure +from taskflow.types import notifier from taskflow.utils import deprecation from taskflow.utils import reflection @@ -396,101 +395,8 @@ Failure = deprecation.moved_class(failure.Failure, 'Failure', __name__, version="0.5", removal_version="?") -class Notifier(object): - """A notification helper class. - - It is intended to be used to subscribe to notifications of events - occurring as well as allow a entity to post said notifications to any - associated subscribers without having either entity care about how this - notification occurs. - """ - - #: Keys that can not be used in callbacks arguments - RESERVED_KEYS = ('details',) - - #: Kleene star constant that is used to recieve all notifications - ANY = '*' - - def __init__(self): - self._listeners = collections.defaultdict(list) - - def __len__(self): - """Returns how many callbacks are registered.""" - count = 0 - for (_event_type, callbacks) in six.iteritems(self._listeners): - count += len(callbacks) - return count - - def is_registered(self, event_type, callback): - """Check if a callback is registered.""" - listeners = list(self._listeners.get(event_type, [])) - for (cb, _args, _kwargs) in listeners: - if reflection.is_same_callback(cb, callback): - return True - return False - - def reset(self): - """Forget all previously registered callbacks.""" - self._listeners.clear() - - def notify(self, event_type, details): - """Notify about event occurrence. - - All callbacks registered to receive notifications about given - event type will be called. - - :param event_type: event type that occurred - :param details: addition event details - """ - listeners = list(self._listeners.get(self.ANY, [])) - for i in self._listeners[event_type]: - if i not in listeners: - listeners.append(i) - if not listeners: - return - for (callback, args, kwargs) in listeners: - if args is None: - args = [] - if kwargs is None: - kwargs = {} - kwargs['details'] = details - try: - callback(event_type, *args, **kwargs) - except Exception: - LOG.warn("Failure calling callback %s to notify about event" - " %s, details: %s", callback, event_type, - details, exc_info=True) - - def register(self, event_type, callback, args=None, kwargs=None): - """Register a callback to be called when event of a given type occurs. - - Callback will be called with provided ``args`` and ``kwargs`` and - when event type occurs (or on any event if ``event_type`` equals to - :attr:`.ANY`). It will also get additional keyword argument, - ``details``, that will hold event details provided to the - :meth:`.notify` method. - """ - assert six.callable(callback), "Callback must be callable" - if self.is_registered(event_type, callback): - raise ValueError("Callback %s already registered" % (callback)) - if kwargs: - for k in self.RESERVED_KEYS: - if k in kwargs: - raise KeyError(("Reserved key '%s' not allowed in " - "kwargs") % k) - kwargs = copy.copy(kwargs) - if args: - args = copy.copy(args) - self._listeners[event_type].append((callback, args, kwargs)) - - def deregister(self, event_type, callback): - """Remove a single callback from listening to event ``event_type``.""" - if event_type not in self._listeners: - return - for i, (cb, args, kwargs) in enumerate(self._listeners[event_type]): - if reflection.is_same_callback(cb, callback): - self._listeners[event_type].pop(i) - break +Notifier = deprecation.moved_class(notifier.Notifier, 'Notifier', __name__, + version="0.5", removal_version="?") @contextlib.contextmanager From d638c8fc7384a214608313b55d51dacd0823e60f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 18 Oct 2014 12:56:28 -0700 Subject: [PATCH 072/240] Bump up the sqlalchemy version for py26 The redhat repo for RDO and epel itself has access to 0.8 so it would be great if we also tested those same versions since those versions are common on that distribution (the main one that still runs py26). Change-Id: I25fe6a345bf417fc6957f075ac01f361bcd36982 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2a09a274..a295abfd 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,7 @@ deps = {[testenv]deps} -r{toxinidir}/requirements-py2.txt MySQL-python eventlet>=0.15.1 - SQLAlchemy>=0.7.8,<=0.7.99 + SQLAlchemy>=0.7.8,<=0.8.99 [testenv:py27] deps = {[testenv]deps} From 3a8a78ee647e43fa0386d4310a384e374d4ce0fa Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 19 Jul 2014 16:52:13 -0700 Subject: [PATCH 073/240] Use constants for link metadata keys Instead of using strings (which can be easy to mistype and get wrong), provide a set of constants that can be used to attach and use these keys in flows and at compilation. This also helps make it more clear what the keys do and where they are used. Change-Id: I5283b27617961136a4582bbcfce4617f05e8dd1d --- taskflow/engines/action_engine/compiler.py | 18 +++++++++--------- taskflow/flow.py | 13 +++++++++++++ taskflow/patterns/graph_flow.py | 8 ++++---- taskflow/patterns/linear_flow.py | 2 +- 4 files changed, 27 insertions(+), 14 deletions(-) diff --git a/taskflow/engines/action_engine/compiler.py b/taskflow/engines/action_engine/compiler.py index 41fe5f96..47bc4a0b 100644 --- a/taskflow/engines/action_engine/compiler.py +++ b/taskflow/engines/action_engine/compiler.py @@ -28,6 +28,11 @@ from taskflow.utils import misc LOG = logging.getLogger(__name__) +_RETRY_EDGE_DATA = { + flow.LINK_RETRY: True, +} +_EDGE_INVARIANTS = (flow.LINK_INVARIANT, flow.LINK_MANUAL, flow.LINK_RETRY) + class Compilation(object): """The result of a compilers compile() is this *immutable* object.""" @@ -45,11 +50,6 @@ class Compilation(object): return self._hierarchy -_RETRY_EDGE_DATA = { - 'retry': True, -} - - class PatternCompiler(object): """Compiles a pattern (or task) into a compilation unit.""" @@ -110,8 +110,8 @@ class PatternCompiler(object): # Add association for each node of graph that has no existing retry. for n in graph.nodes_iter(): - if n is not retry and 'retry' not in graph.node[n]: - graph.node[n]['retry'] = retry + if n is not retry and flow.LINK_RETRY not in graph.node[n]: + graph.node[n][flow.LINK_RETRY] = retry def _flatten_task(self, task, parent): """Flattens a individual task.""" @@ -143,7 +143,7 @@ class PatternCompiler(object): for (u, v, attrs) in flow.iter_links(): u_g = subgraphs[u] v_g = subgraphs[v] - if any(attrs.get(k) for k in ('invariant', 'manual', 'retry')): + if any(attrs.get(k) for k in _EDGE_INVARIANTS): # Connect nodes with no predecessors in v to nodes with # no successors in u (thus maintaining the edge dependency). self._add_new_edges(graph, @@ -151,7 +151,7 @@ class PatternCompiler(object): v_g.no_predecessors_iter(), edge_attrs=attrs) else: - # This is dependency-only edge, connect corresponding + # This is symbol dependency edge, connect corresponding # providers and consumers. for provider in u_g: for consumer in v_g: diff --git a/taskflow/flow.py b/taskflow/flow.py index 0359125d..c5fb4296 100644 --- a/taskflow/flow.py +++ b/taskflow/flow.py @@ -20,6 +20,19 @@ import six from taskflow.utils import reflection +# Link metadata keys that have inherent/special meaning. +# +# This key denotes the link is an invariant that ensures the order is +# correctly preserved. +LINK_INVARIANT = 'invariant' +# This key denotes the link is a manually/user-specified. +LINK_MANUAL = 'manual' +# This key denotes the link was created when resolving/compiling retries. +LINK_RETRY = 'retry' +# This key denotes the link was created due to symbol constraints and the +# value will be a set of names that the constraint ensures are satisfied. +LINK_REASONS = 'reasons' + @six.add_metaclass(abc.ABCMeta) class Flow(object): diff --git a/taskflow/patterns/graph_flow.py b/taskflow/patterns/graph_flow.py index f07e7435..f71f285b 100644 --- a/taskflow/patterns/graph_flow.py +++ b/taskflow/patterns/graph_flow.py @@ -75,11 +75,11 @@ class Flow(flow.Flow): if not attrs: attrs = {} if manual: - attrs['manual'] = True + attrs[flow.LINK_MANUAL] = True if reason is not None: - if 'reasons' not in attrs: - attrs['reasons'] = set() - attrs['reasons'].add(reason) + if flow.LINK_REASONS not in attrs: + attrs[flow.LINK_REASONS] = set() + attrs[flow.LINK_REASONS].add(reason) if not mutable_graph: graph = gr.DiGraph(graph) graph.add_edge(u, v, **attrs) diff --git a/taskflow/patterns/linear_flow.py b/taskflow/patterns/linear_flow.py index 77559231..3067076c 100644 --- a/taskflow/patterns/linear_flow.py +++ b/taskflow/patterns/linear_flow.py @@ -17,7 +17,7 @@ from taskflow import flow -_LINK_METADATA = {'invariant': True} +_LINK_METADATA = {flow.LINK_INVARIANT: True} class Flow(flow.Flow): From d433a5323ff4fbf1d973ca7605ac62819c19039a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 16 Sep 2014 22:46:39 -0700 Subject: [PATCH 074/240] Deprecate `engine_conf` and prefer `engine` instead To avoid having one set of options coming from `engine_conf` and another set of options coming from `kwargs` and another set coming from `engine_conf` if it is a URI just start to shift toward `engine_conf` being deprecated and `engine` being a string type only (or a URI with additional query parameters) and having any additional **kwargs that are provided just get merged into the final engine options. This adds a new helper function that handles all these various options and adds in a keyword argument `engine` that will be shifted to in a future version (in that future version we can also then remove the `engine_conf` and just stick to a smaller set of option mechanisms). It also adjusts all examples to use this new and more easier to understand format and adjusts tests, conductor interface to use this new more easily understandable style of getting an engine. Change-Id: Ic7617057338e0c63775cf38a24643cff6e454950 --- doc/source/engines.rst | 13 +- doc/source/workers.rst | 30 ++-- taskflow/conductors/base.py | 19 ++- taskflow/conductors/single_threaded.py | 10 +- taskflow/engines/action_engine/engine.py | 16 +- taskflow/engines/base.py | 13 +- taskflow/engines/helpers.py | 161 ++++++++++-------- taskflow/engines/worker_based/engine.py | 30 ++-- taskflow/examples/calculate_in_parallel.py | 2 +- taskflow/examples/create_parallel_volume.py | 10 +- taskflow/examples/delayed_return.py | 2 +- taskflow/examples/fake_billing.py | 2 +- taskflow/examples/graph_flow.py | 4 +- taskflow/examples/persistence_example.py | 13 +- taskflow/examples/resume_vm_boot.py | 14 +- taskflow/examples/resume_volume_create.py | 6 +- taskflow/examples/wbe_simple_linear.py | 14 +- taskflow/examples/wrapped_exception.py | 2 +- .../tests/unit/conductor/test_conductor.py | 7 +- taskflow/tests/unit/test_arguments_passing.py | 5 +- taskflow/tests/unit/test_engine_helpers.py | 21 ++- taskflow/tests/unit/test_engines.py | 29 ++-- taskflow/tests/unit/test_retries.py | 5 +- taskflow/tests/unit/test_suspend_flow.py | 8 +- 24 files changed, 209 insertions(+), 227 deletions(-) diff --git a/doc/source/engines.rst b/doc/source/engines.rst index be19aa45..e3680672 100644 --- a/doc/source/engines.rst +++ b/doc/source/engines.rst @@ -118,9 +118,9 @@ might look like:: ... flow = make_flow() - engine = engines.load(flow, engine_conf=my_conf, - backend=my_persistence_conf) - engine.run + eng = engines.load(flow, engine='serial', backend=my_persistence_conf) + eng.run() + ... .. automodule:: taskflow.engines.helpers @@ -129,11 +129,8 @@ Usage ===== To select which engine to use and pass parameters to an engine you should use -the ``engine_conf`` parameter any helper factory function accepts. It may be: - -* A string, naming the engine type. -* A dictionary, naming engine type with key ``'engine'`` and possibly - type-specific engine configuration parameters. +the ``engine`` parameter any engine helper function accepts and for any engine +specific options use the ``kwargs`` parameter. Types ===== diff --git a/doc/source/workers.rst b/doc/source/workers.rst index 9c2f2b9c..caac6aa0 100644 --- a/doc/source/workers.rst +++ b/doc/source/workers.rst @@ -273,32 +273,26 @@ For complete parameters and object usage please see .. code:: python - engine_conf = { - 'engine': 'worker-based', - 'url': 'amqp://guest:guest@localhost:5672//', - 'exchange': 'test-exchange', - 'topics': ['topic1', 'topic2'], - } flow = lf.Flow('simple-linear').add(...) - eng = taskflow.engines.load(flow, engine_conf=engine_conf) + eng = taskflow.engines.load(flow, engine='worker-based', + url='amqp://guest:guest@localhost:5672//', + exchange='test-exchange', + topics=['topic1', 'topic2']) eng.run() **Example with filesystem transport:** .. code:: python - engine_conf = { - 'engine': 'worker-based', - 'exchange': 'test-exchange', - 'topics': ['topic1', 'topic2'], - 'transport': 'filesystem', - 'transport_options': { - 'data_folder_in': '/tmp/test', - 'data_folder_out': '/tmp/test', - }, - } flow = lf.Flow('simple-linear').add(...) - eng = taskflow.engines.load(flow, engine_conf=engine_conf) + eng = taskflow.engines.load(flow, engine='worker-based', + exchange='test-exchange', + topics=['topic1', 'topic2'], + transport='filesystem', + transport_options={ + 'data_folder_in': '/tmp/in', + 'data_folder_out': '/tmp/out', + }) eng.run() Additional supported keyword arguments: diff --git a/taskflow/conductors/base.py b/taskflow/conductors/base.py index c881346e..f7546c3e 100644 --- a/taskflow/conductors/base.py +++ b/taskflow/conductors/base.py @@ -17,7 +17,7 @@ import threading import six -import taskflow.engines +from taskflow import engines from taskflow import exceptions as excp from taskflow.utils import lock_utils @@ -34,10 +34,15 @@ class Conductor(object): period of time will finish up the prior failed conductors work. """ - def __init__(self, name, jobboard, engine_conf, persistence): + def __init__(self, name, jobboard, persistence, + engine=None, engine_options=None): self._name = name self._jobboard = jobboard - self._engine_conf = engine_conf + self._engine = engine + if not engine_options: + self._engine_options = {} + else: + self._engine_options = engine_options.copy() self._persistence = persistence self._lock = threading.RLock() @@ -83,10 +88,10 @@ class Conductor(object): store = dict(job.details["store"]) else: store = {} - return taskflow.engines.load_from_detail(flow_detail, - store=store, - engine_conf=self._engine_conf, - backend=self._persistence) + return engines.load_from_detail(flow_detail, store=store, + engine=self._engine, + backend=self._persistence, + **self._engine_options) @lock_utils.locked def connect(self): diff --git a/taskflow/conductors/single_threaded.py b/taskflow/conductors/single_threaded.py index 23994e79..84038ef1 100644 --- a/taskflow/conductors/single_threaded.py +++ b/taskflow/conductors/single_threaded.py @@ -51,11 +51,11 @@ class SingleThreadedConductor(base.Conductor): upon the jobboard capabilities to automatically abandon these jobs. """ - def __init__(self, name, jobboard, engine_conf, persistence, - wait_timeout=None): - super(SingleThreadedConductor, self).__init__(name, jobboard, - engine_conf, - persistence) + def __init__(self, name, jobboard, persistence, + engine=None, engine_options=None, wait_timeout=None): + super(SingleThreadedConductor, self).__init__( + name, jobboard, persistence, + engine=engine, engine_options=engine_options) if wait_timeout is None: wait_timeout = WAIT_TIMEOUT if isinstance(wait_timeout, (int, float) + six.string_types): diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index 69f05bd3..22db3ed7 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -59,8 +59,8 @@ class ActionEngine(base.EngineBase): _compiler_factory = compiler.PatternCompiler _task_executor_factory = executor.SerialTaskExecutor - def __init__(self, flow, flow_detail, backend, conf): - super(ActionEngine, self).__init__(flow, flow_detail, backend, conf) + def __init__(self, flow, flow_detail, backend, options): + super(ActionEngine, self).__init__(flow, flow_detail, backend, options) self._runtime = None self._compiled = False self._compilation = None @@ -230,12 +230,6 @@ class ParallelActionEngine(ActionEngine): _storage_factory = atom_storage.MultiThreadedStorage def _task_executor_factory(self): - return executor.ParallelTaskExecutor(executor=self._executor, - max_workers=self._max_workers) - - def __init__(self, flow, flow_detail, backend, conf, - executor=None, max_workers=None): - super(ParallelActionEngine, self).__init__(flow, flow_detail, - backend, conf) - self._executor = executor - self._max_workers = max_workers + return executor.ParallelTaskExecutor( + executor=self._options.get('executor'), + max_workers=self._options.get('max_workers')) diff --git a/taskflow/engines/base.py b/taskflow/engines/base.py index 4bfcbabc..63389fe0 100644 --- a/taskflow/engines/base.py +++ b/taskflow/engines/base.py @@ -32,17 +32,22 @@ class EngineBase(object): occur related to the tasks the engine contains. """ - def __init__(self, flow, flow_detail, backend, conf): + def __init__(self, flow, flow_detail, backend, options): self._flow = flow self._flow_detail = flow_detail self._backend = backend - if not conf: - self._conf = {} + if not options: + self._options = {} else: - self._conf = dict(conf) + self._options = dict(options) self.notifier = misc.Notifier() self.task_notifier = misc.Notifier() + @property + def options(self): + """The options that were passed to this engine on construction.""" + return self._options + @misc.cachedproperty def storage(self): """The storage unit for this flow.""" diff --git a/taskflow/engines/helpers.py b/taskflow/engines/helpers.py index bfbaaa53..a6236591 100644 --- a/taskflow/engines/helpers.py +++ b/taskflow/engines/helpers.py @@ -15,6 +15,7 @@ # under the License. import contextlib +import warnings from oslo.utils import importutils import six @@ -30,6 +31,40 @@ from taskflow.utils import reflection # NOTE(imelnikov): this is the entrypoint namespace, not the module namespace. ENGINES_NAMESPACE = 'taskflow.engines' +# The default entrypoint engine type looked for when it is not provided. +ENGINE_DEFAULT = 'default' + + +def _extract_engine(**kwargs): + """Extracts the engine kind and any associated options.""" + options = {} + kind = kwargs.pop('engine', None) + engine_conf = kwargs.pop('engine_conf', None) + if engine_conf is not None: + warnings.warn("Using the 'engine_conf' argument is" + " deprecated and will be removed in a future version," + " please use the 'engine' argument instead.", + DeprecationWarning) + if isinstance(engine_conf, six.string_types): + kind = engine_conf + else: + options.update(engine_conf) + kind = options.pop('engine', None) + if not kind: + kind = ENGINE_DEFAULT + # See if it's a URI and if so, extract any further options... + try: + pieces = misc.parse_uri(kind) + except (TypeError, ValueError): + pass + else: + kind = pieces['scheme'] + options = misc.merge_uri(pieces, options.copy()) + # Merge in any leftover **kwargs into the options, this makes it so that + # the provided **kwargs override any URI or engine_conf specific options. + options.update(kwargs) + return (kind, options) + def _fetch_factory(factory_name): try: @@ -56,49 +91,43 @@ def _fetch_validate_factory(flow_factory): def load(flow, store=None, flow_detail=None, book=None, - engine_conf=None, backend=None, namespace=ENGINES_NAMESPACE, - **kwargs): + engine_conf=None, backend=None, + namespace=ENGINES_NAMESPACE, engine=ENGINE_DEFAULT, **kwargs): """Load a flow into an engine. - This function creates and prepares engine to run the - flow. All that is left is to run the engine with 'run()' method. + This function creates and prepares an engine to run the provided flow. All + that is left after this returns is to run the engine with the + engines ``run()`` method. - Which engine to load is specified in 'engine_conf' parameter. It - can be a string that names engine type or a dictionary which holds - engine type (with 'engine' key) and additional engine-specific - configuration. + Which engine to load is specified via the ``engine`` parameter. It + can be a string that names the engine type to use, or a string that + is a URI with a scheme that names the engine type to use and further + options contained in the URI's host, port, and query parameters... - Which storage backend to use is defined by backend parameter. It + Which storage backend to use is defined by the backend parameter. It can be backend itself, or a dictionary that is passed to - taskflow.persistence.backends.fetch to obtain backend. + ``taskflow.persistence.backends.fetch()`` to obtain a viable backend. :param flow: flow to load :param store: dict -- data to put to storage to satisfy flow requirements :param flow_detail: FlowDetail that holds the state of the flow (if one is not provided then one will be created for you in the provided backend) :param book: LogBook to create flow detail in if flow_detail is None - :param engine_conf: engine type and configuration configuration - :param backend: storage backend to use or configuration - :param namespace: driver namespace for stevedore (default is fine - if you don't know what is it) + :param engine_conf: engine type or URI and options (**deprecated**) + :param backend: storage backend to use or configuration that defines it + :param namespace: driver namespace for stevedore (or empty for default) + :param engine: string engine type or URI string with scheme that contains + the engine type and any URI specific components that will + become part of the engine options. + :param kwargs: arbitrary keyword arguments passed as options (merged with + any extracted ``engine`` and ``engine_conf`` options), + typically used for any engine specific options that do not + fit as any of the existing arguments. :returns: engine """ - if engine_conf is None: - engine_conf = {'engine': 'default'} - - # NOTE(imelnikov): this allows simpler syntax. - if isinstance(engine_conf, six.string_types): - engine_conf = {'engine': engine_conf} - - engine_name = engine_conf['engine'] - try: - pieces = misc.parse_uri(engine_name) - except (TypeError, ValueError): - pass - else: - engine_name = pieces['scheme'] - engine_conf = misc.merge_uri(pieces, engine_conf.copy()) + kind, options = _extract_engine(engine_conf=engine_conf, + engine=engine, **kwargs) if isinstance(backend, dict): backend = p_backends.fetch(backend) @@ -109,13 +138,12 @@ def load(flow, store=None, flow_detail=None, book=None, try: mgr = stevedore.driver.DriverManager( - namespace, engine_name, + namespace, kind, invoke_on_load=True, - invoke_args=(flow, flow_detail, backend, engine_conf), - invoke_kwds=kwargs) + invoke_args=(flow, flow_detail, backend, options)) engine = mgr.driver except RuntimeError as e: - raise exc.NotFound("Could not find engine %s" % (engine_name), e) + raise exc.NotFound("Could not find engine '%s'" % (kind), e) else: if store: engine.storage.inject(store) @@ -123,35 +151,20 @@ def load(flow, store=None, flow_detail=None, book=None, def run(flow, store=None, flow_detail=None, book=None, - engine_conf=None, backend=None, namespace=ENGINES_NAMESPACE, **kwargs): + engine_conf=None, backend=None, namespace=ENGINES_NAMESPACE, + engine=ENGINE_DEFAULT, **kwargs): """Run the flow. - This function load the flow into engine (with 'load' function) - and runs the engine. + This function loads the flow into an engine (with the :func:`load() ` + function) and runs the engine. - Which engine to load is specified in 'engine_conf' parameter. It - can be a string that names engine type or a dictionary which holds - engine type (with 'engine' key) and additional engine-specific - configuration. + The arguments are interpreted as for :func:`load() `. - Which storage backend to use is defined by backend parameter. It - can be backend itself, or a dictionary that is passed to - taskflow.persistence.backends.fetch to obtain backend. - - :param flow: flow to run - :param store: dict -- data to put to storage to satisfy flow requirements - :param flow_detail: FlowDetail that holds the state of the flow (if one is - not provided then one will be created for you in the provided backend) - :param book: LogBook to create flow detail in if flow_detail is None - :param engine_conf: engine type and configuration configuration - :param backend: storage backend to use or configuration - :param namespace: driver namespace for stevedore (default is fine - if you don't know what is it) - :returns: dictionary of all named task results (see Storage.fetch_all) + :returns: dictionary of all named results (see ``storage.fetch_all()``) """ engine = load(flow, store=store, flow_detail=flow_detail, book=book, engine_conf=engine_conf, backend=backend, - namespace=namespace, **kwargs) + namespace=namespace, engine=engine, **kwargs) engine.run() return engine.storage.fetch_all() @@ -196,23 +209,21 @@ def save_factory_details(flow_detail, def load_from_factory(flow_factory, factory_args=None, factory_kwargs=None, store=None, book=None, engine_conf=None, backend=None, - namespace=ENGINES_NAMESPACE, **kwargs): + namespace=ENGINES_NAMESPACE, engine=ENGINE_DEFAULT, + **kwargs): """Loads a flow from a factory function into an engine. Gets flow factory function (or name of it) and creates flow with - it. Then, flow is loaded into engine with load(), and factory - function fully qualified name is saved to flow metadata so that - it can be later resumed with resume. + it. Then, the flow is loaded into an engine with the :func:`load() ` + function, and the factory function fully qualified name is saved to flow + metadata so that it can be later resumed. :param flow_factory: function or string: function that creates the flow :param factory_args: list or tuple of factory positional arguments :param factory_kwargs: dict of factory keyword arguments - :param store: dict -- data to put to storage to satisfy flow requirements - :param book: LogBook to create flow detail in - :param engine_conf: engine type and configuration configuration - :param backend: storage backend to use or configuration - :param namespace: driver namespace for stevedore (default is fine - if you don't know what is it) + + Further arguments are interpreted as for :func:`load() `. + :returns: engine """ @@ -230,7 +241,7 @@ def load_from_factory(flow_factory, factory_args=None, factory_kwargs=None, backend=backend) return load(flow=flow, store=store, flow_detail=flow_detail, book=book, engine_conf=engine_conf, backend=backend, namespace=namespace, - **kwargs) + engine=engine, **kwargs) def flow_from_detail(flow_detail): @@ -261,21 +272,21 @@ def flow_from_detail(flow_detail): def load_from_detail(flow_detail, store=None, engine_conf=None, backend=None, - namespace=ENGINES_NAMESPACE, **kwargs): + namespace=ENGINES_NAMESPACE, engine=ENGINE_DEFAULT, + **kwargs): """Reloads an engine previously saved. - This reloads the flow using the flow_from_detail() function and then calls - into the load() function to create an engine from that flow. + This reloads the flow using the + :func:`flow_from_detail() ` function and then calls + into the :func:`load() ` function to create an engine from that flow. :param flow_detail: FlowDetail that holds state of the flow to load - :param store: dict -- data to put to storage to satisfy flow requirements - :param engine_conf: engine type and configuration configuration - :param backend: storage backend to use or configuration - :param namespace: driver namespace for stevedore (default is fine - if you don't know what is it) + + Further arguments are interpreted as for :func:`load() `. + :returns: engine """ flow = flow_from_detail(flow_detail) return load(flow, flow_detail=flow_detail, store=store, engine_conf=engine_conf, backend=backend, - namespace=namespace, **kwargs) + namespace=namespace, engine=engine, **kwargs) diff --git a/taskflow/engines/worker_based/engine.py b/taskflow/engines/worker_based/engine.py index 0c1b8ead..aefce23f 100644 --- a/taskflow/engines/worker_based/engine.py +++ b/taskflow/engines/worker_based/engine.py @@ -23,7 +23,7 @@ from taskflow import storage as t_storage class WorkerBasedActionEngine(engine.ActionEngine): """Worker based action engine. - Specific backend configuration: + Specific backend options: :param exchange: broker exchange exchange name in which executor / worker communication is performed @@ -45,19 +45,15 @@ class WorkerBasedActionEngine(engine.ActionEngine): _storage_factory = t_storage.SingleThreadedStorage def _task_executor_factory(self): - if self._executor is not None: - return self._executor - return executor.WorkerTaskExecutor( - uuid=self._flow_detail.uuid, - url=self._conf.get('url'), - exchange=self._conf.get('exchange', 'default'), - topics=self._conf.get('topics', []), - transport=self._conf.get('transport'), - transport_options=self._conf.get('transport_options'), - transition_timeout=self._conf.get('transition_timeout', - pr.REQUEST_TIMEOUT)) - - def __init__(self, flow, flow_detail, backend, conf, **kwargs): - super(WorkerBasedActionEngine, self).__init__( - flow, flow_detail, backend, conf) - self._executor = kwargs.get('executor') + try: + return self._options['executor'] + except KeyError: + return executor.WorkerTaskExecutor( + uuid=self._flow_detail.uuid, + url=self._options.get('url'), + exchange=self._options.get('exchange', 'default'), + topics=self._options.get('topics', []), + transport=self._options.get('transport'), + transport_options=self._options.get('transport_options'), + transition_timeout=self._options.get('transition_timeout', + pr.REQUEST_TIMEOUT)) diff --git a/taskflow/examples/calculate_in_parallel.py b/taskflow/examples/calculate_in_parallel.py index 0215f956..7ab32fae 100644 --- a/taskflow/examples/calculate_in_parallel.py +++ b/taskflow/examples/calculate_in_parallel.py @@ -93,5 +93,5 @@ flow = lf.Flow('root').add( # The result here will be all results (from all tasks) which is stored in an # in-memory storage location that backs this engine since it is not configured # with persistence storage. -result = taskflow.engines.run(flow, engine_conf='parallel') +result = taskflow.engines.run(flow, engine='parallel') print(result) diff --git a/taskflow/examples/create_parallel_volume.py b/taskflow/examples/create_parallel_volume.py index de511adf..5185330b 100644 --- a/taskflow/examples/create_parallel_volume.py +++ b/taskflow/examples/create_parallel_volume.py @@ -64,13 +64,9 @@ VOLUME_COUNT = 5 # time difference that this causes. SERIAL = False if SERIAL: - engine_conf = { - 'engine': 'serial', - } + engine = 'serial' else: - engine_conf = { - 'engine': 'parallel', - } + engine = 'parallel' class VolumeCreator(task.Task): @@ -106,7 +102,7 @@ for i in range(0, VOLUME_COUNT): # Show how much time the overall engine loading and running takes. with show_time(name=flow.name.title()): - eng = engines.load(flow, engine_conf=engine_conf) + eng = engines.load(flow, engine=engine) # This context manager automatically adds (and automatically removes) a # helpful set of state transition notification printing helper utilities # that show you exactly what transitions the engine is going through diff --git a/taskflow/examples/delayed_return.py b/taskflow/examples/delayed_return.py index 46578621..bc44a897 100644 --- a/taskflow/examples/delayed_return.py +++ b/taskflow/examples/delayed_return.py @@ -74,7 +74,7 @@ class Bye(task.Task): def return_from_flow(pool): wf = lf.Flow("root").add(Hi("hi"), Bye("bye")) - eng = taskflow.engines.load(wf, engine_conf='serial') + eng = taskflow.engines.load(wf, engine='serial') f = futures.Future() watcher = PokeFutureListener(eng, f, 'hi') watcher.register() diff --git a/taskflow/examples/fake_billing.py b/taskflow/examples/fake_billing.py index 9a421f92..ac15dbae 100644 --- a/taskflow/examples/fake_billing.py +++ b/taskflow/examples/fake_billing.py @@ -170,7 +170,7 @@ flow.add(sub_flow) store = { 'request': misc.AttrDict(user="bob", id="1.35"), } -eng = engines.load(flow, engine_conf='serial', store=store) +eng = engines.load(flow, engine='serial', store=store) # This context manager automatically adds (and automatically removes) a # helpful set of state transition notification printing helper utilities diff --git a/taskflow/examples/graph_flow.py b/taskflow/examples/graph_flow.py index 99dfdd45..9f28dc71 100644 --- a/taskflow/examples/graph_flow.py +++ b/taskflow/examples/graph_flow.py @@ -81,11 +81,11 @@ store = { } result = taskflow.engines.run( - flow, engine_conf='serial', store=store) + flow, engine='serial', store=store) print("Single threaded engine result %s" % result) result = taskflow.engines.run( - flow, engine_conf='parallel', store=store) + flow, engine='parallel', store=store) print("Multi threaded engine result %s" % result) diff --git a/taskflow/examples/persistence_example.py b/taskflow/examples/persistence_example.py index 720914cd..fe5968fe 100644 --- a/taskflow/examples/persistence_example.py +++ b/taskflow/examples/persistence_example.py @@ -91,20 +91,15 @@ else: blowup = True with eu.get_backend(backend_uri) as backend: - # Now we can run. - engine_config = { - 'backend': backend, - 'engine_conf': 'serial', - 'book': logbook.LogBook("my-test"), - } - # Make a flow that will blowup if the file doesn't exist previously, if it # did exist, assume we won't blowup (and therefore this shows the undo # and redo that a flow will go through). + book = logbook.LogBook("my-test") flow = make_flow(blowup=blowup) eu.print_wrapped("Running") try: - eng = engines.load(flow, **engine_config) + eng = engines.load(flow, engine='serial', + backend=backend, book=book) eng.run() if not blowup: eu.rm_path(persist_path) @@ -115,4 +110,4 @@ with eu.get_backend(backend_uri) as backend: traceback.print_exc(file=sys.stdout) eu.print_wrapped("Book contents") - print(p_utils.pformat(engine_config['book'])) + print(p_utils.pformat(book)) diff --git a/taskflow/examples/resume_vm_boot.py b/taskflow/examples/resume_vm_boot.py index acdf42b5..203cb882 100644 --- a/taskflow/examples/resume_vm_boot.py +++ b/taskflow/examples/resume_vm_boot.py @@ -235,11 +235,9 @@ with eu.get_backend() as backend: flow_id = None # Set up how we want our engine to run, serial, parallel... - engine_conf = { - 'engine': 'parallel', - } + executor = None if e_utils.EVENTLET_AVAILABLE: - engine_conf['executor'] = e_utils.GreenExecutor(5) + executor = e_utils.GreenExecutor(5) # Create/fetch a logbook that will track the workflows work. book = None @@ -255,15 +253,15 @@ with eu.get_backend() as backend: book = p_utils.temporary_log_book(backend) engine = engines.load_from_factory(create_flow, backend=backend, book=book, - engine_conf=engine_conf) + engine='parallel', + executor=executor) print("!! Your tracking id is: '%s+%s'" % (book.uuid, engine.storage.flow_uuid)) print("!! Please submit this on later runs for tracking purposes") else: # Attempt to load from a previously partially completed flow. - engine = engines.load_from_detail(flow_detail, - backend=backend, - engine_conf=engine_conf) + engine = engines.load_from_detail(flow_detail, backend=backend, + engine='parallel', executor=executor) # Make me my vm please! eu.print_wrapped('Running') diff --git a/taskflow/examples/resume_volume_create.py b/taskflow/examples/resume_volume_create.py index 0fe502e4..275fa6b8 100644 --- a/taskflow/examples/resume_volume_create.py +++ b/taskflow/examples/resume_volume_create.py @@ -143,13 +143,9 @@ with example_utils.get_backend() as backend: flow_detail = find_flow_detail(backend, book_id, flow_id) # Load and run. - engine_conf = { - 'engine': 'serial', - } engine = engines.load(flow, flow_detail=flow_detail, - backend=backend, - engine_conf=engine_conf) + backend=backend, engine='serial') engine.run() # How to use. diff --git a/taskflow/examples/wbe_simple_linear.py b/taskflow/examples/wbe_simple_linear.py index bfec2d86..a15b48fa 100644 --- a/taskflow/examples/wbe_simple_linear.py +++ b/taskflow/examples/wbe_simple_linear.py @@ -69,19 +69,16 @@ WORKER_CONF = { 'taskflow.tests.utils:TaskMultiArgOneReturn' ], } -ENGINE_CONF = { - 'engine': 'worker-based', -} -def run(engine_conf): +def run(engine_options): flow = lf.Flow('simple-linear').add( utils.TaskOneArgOneReturn(provides='result1'), utils.TaskMultiArgOneReturn(provides='result2') ) eng = engines.load(flow, store=dict(x=111, y=222, z=333), - engine_conf=engine_conf) + engine='worker-based', **engine_options) eng.run() return eng.storage.fetch_all() @@ -115,8 +112,7 @@ if __name__ == "__main__": }) worker_conf = dict(WORKER_CONF) worker_conf.update(shared_conf) - engine_conf = dict(ENGINE_CONF) - engine_conf.update(shared_conf) + engine_options = dict(shared_conf) workers = [] worker_topics = [] @@ -135,8 +131,8 @@ if __name__ == "__main__": # Now use those workers to do something. print('Executing some work.') - engine_conf['topics'] = worker_topics - result = run(engine_conf) + engine_options['topics'] = worker_topics + result = run(engine_options) print('Execution finished.') # This is done so that the test examples can work correctly # even when the keys change order (which will happen in various diff --git a/taskflow/examples/wrapped_exception.py b/taskflow/examples/wrapped_exception.py index 7679a150..dff6b2b4 100644 --- a/taskflow/examples/wrapped_exception.py +++ b/taskflow/examples/wrapped_exception.py @@ -93,7 +93,7 @@ def run(**store): try: with utils.wrap_all_failures(): taskflow.engines.run(flow, store=store, - engine_conf='parallel') + engine='parallel') except exceptions.WrappedFailure as ex: unknown_failures = [] for failure in ex: diff --git a/taskflow/tests/unit/conductor/test_conductor.py b/taskflow/tests/unit/conductor/test_conductor.py index b43ba035..216fa387 100644 --- a/taskflow/tests/unit/conductor/test_conductor.py +++ b/taskflow/tests/unit/conductor/test_conductor.py @@ -63,11 +63,8 @@ class SingleThreadedConductorTest(test_utils.EngineTestBase, test.TestCase): board = impl_zookeeper.ZookeeperJobBoard(name, {}, client=client, persistence=persistence) - engine_conf = { - 'engine': 'default', - } - conductor = stc.SingleThreadedConductor(name, board, engine_conf, - persistence, wait_timeout) + conductor = stc.SingleThreadedConductor(name, board, persistence, + wait_timeout=wait_timeout) return misc.AttrDict(board=board, client=client, persistence=persistence, diff --git a/taskflow/tests/unit/test_arguments_passing.py b/taskflow/tests/unit/test_arguments_passing.py index 5e9fc3a8..fb4744bd 100644 --- a/taskflow/tests/unit/test_arguments_passing.py +++ b/taskflow/tests/unit/test_arguments_passing.py @@ -155,15 +155,14 @@ class SingleThreadedEngineTest(ArgumentsPassingTest, def _make_engine(self, flow, flow_detail=None): return taskflow.engines.load(flow, flow_detail=flow_detail, - engine_conf='serial', + engine='serial', backend=self.backend) class MultiThreadedEngineTest(ArgumentsPassingTest, test.TestCase): def _make_engine(self, flow, flow_detail=None, executor=None): - engine_conf = dict(engine='parallel') return taskflow.engines.load(flow, flow_detail=flow_detail, - engine_conf=engine_conf, + engine='parallel', backend=self.backend, executor=executor) diff --git a/taskflow/tests/unit/test_engine_helpers.py b/taskflow/tests/unit/test_engine_helpers.py index 2353d77d..4087d839 100644 --- a/taskflow/tests/unit/test_engine_helpers.py +++ b/taskflow/tests/unit/test_engine_helpers.py @@ -24,17 +24,30 @@ from taskflow.utils import persistence_utils as p_utils class EngineLoadingTestCase(test.TestCase): - def test_default_load(self): + def _make_dummy_flow(self): f = linear_flow.Flow('test') f.add(test_utils.TaskOneReturn("run-1")) + return f + + def test_default_load(self): + f = self._make_dummy_flow() e = taskflow.engines.load(f) self.assertIsNotNone(e) def test_unknown_load(self): - f = linear_flow.Flow('test') - f.add(test_utils.TaskOneReturn("run-1")) + f = self._make_dummy_flow() self.assertRaises(exc.NotFound, taskflow.engines.load, f, - engine_conf='not_really_any_engine') + engine='not_really_any_engine') + + def test_options_empty(self): + f = self._make_dummy_flow() + e = taskflow.engines.load(f) + self.assertEqual({}, e.options) + + def test_options_passthrough(self): + f = self._make_dummy_flow() + e = taskflow.engines.load(f, pass_1=1, pass_2=2) + self.assertEqual({'pass_1': 1, 'pass_2': 2}, e.options) class FlowFromDetailTestCase(test.TestCase): diff --git a/taskflow/tests/unit/test_engines.py b/taskflow/tests/unit/test_engines.py index 243635fb..4ce291d1 100644 --- a/taskflow/tests/unit/test_engines.py +++ b/taskflow/tests/unit/test_engines.py @@ -550,7 +550,7 @@ class SingleThreadedEngineTest(EngineTaskTest, def _make_engine(self, flow, flow_detail=None): return taskflow.engines.load(flow, flow_detail=flow_detail, - engine_conf='serial', + engine='serial', backend=self.backend) def test_correct_load(self): @@ -570,16 +570,14 @@ class MultiThreadedEngineTest(EngineTaskTest, EngineCheckingTaskTest, test.TestCase): def _make_engine(self, flow, flow_detail=None, executor=None): - engine_conf = dict(engine='parallel') return taskflow.engines.load(flow, flow_detail=flow_detail, - engine_conf=engine_conf, backend=self.backend, - executor=executor) + executor=executor, + engine='parallel') def test_correct_load(self): engine = self._make_engine(utils.TaskNoRequiresNoReturns) self.assertIsInstance(engine, eng.ParallelActionEngine) - self.assertIs(engine._executor, None) def test_using_common_executor(self): flow = utils.TaskNoRequiresNoReturns(name='task1') @@ -587,7 +585,7 @@ class MultiThreadedEngineTest(EngineTaskTest, try: e1 = self._make_engine(flow, executor=executor) e2 = self._make_engine(flow, executor=executor) - self.assertIs(e1._executor, e2._executor) + self.assertIs(e1.options['executor'], e2.options['executor']) finally: executor.shutdown(wait=True) @@ -604,11 +602,9 @@ class ParallelEngineWithEventletTest(EngineTaskTest, def _make_engine(self, flow, flow_detail=None, executor=None): if executor is None: executor = eu.GreenExecutor() - engine_conf = dict(engine='parallel', - executor=executor) return taskflow.engines.load(flow, flow_detail=flow_detail, - engine_conf=engine_conf, - backend=self.backend) + backend=self.backend, engine='parallel', + executor=executor) class WorkerBasedEngineTest(EngineTaskTest, @@ -647,15 +643,12 @@ class WorkerBasedEngineTest(EngineTaskTest, super(WorkerBasedEngineTest, self).tearDown() def _make_engine(self, flow, flow_detail=None): - engine_conf = { - 'engine': 'worker-based', - 'exchange': self.exchange, - 'topics': [self.topic], - 'transport': self.transport, - } return taskflow.engines.load(flow, flow_detail=flow_detail, - engine_conf=engine_conf, - backend=self.backend) + backend=self.backend, + engine='worker-based', + exchange=self.exchange, + topics=[self.topic], + transport=self.transport) def test_correct_load(self): engine = self._make_engine(utils.TaskNoRequiresNoReturns) diff --git a/taskflow/tests/unit/test_retries.py b/taskflow/tests/unit/test_retries.py index 71ea70cb..7ba91b45 100644 --- a/taskflow/tests/unit/test_retries.py +++ b/taskflow/tests/unit/test_retries.py @@ -758,7 +758,7 @@ class SingleThreadedEngineTest(RetryTest, def _make_engine(self, flow, flow_detail=None): return taskflow.engines.load(flow, flow_detail=flow_detail, - engine_conf='serial', + engine='serial', backend=self.backend) @@ -766,8 +766,7 @@ class MultiThreadedEngineTest(RetryTest, RetryParallelExecutionTest, test.TestCase): def _make_engine(self, flow, flow_detail=None, executor=None): - engine_conf = dict(engine='parallel') return taskflow.engines.load(flow, flow_detail=flow_detail, - engine_conf=engine_conf, + engine='parallel', backend=self.backend, executor=executor) diff --git a/taskflow/tests/unit/test_suspend_flow.py b/taskflow/tests/unit/test_suspend_flow.py index bb953449..7b9875c9 100644 --- a/taskflow/tests/unit/test_suspend_flow.py +++ b/taskflow/tests/unit/test_suspend_flow.py @@ -168,16 +168,15 @@ class SingleThreadedEngineTest(SuspendFlowTest, def _make_engine(self, flow, flow_detail=None): return taskflow.engines.load(flow, flow_detail=flow_detail, - engine_conf='serial', + engine='serial', backend=self.backend) class MultiThreadedEngineTest(SuspendFlowTest, test.TestCase): def _make_engine(self, flow, flow_detail=None, executor=None): - engine_conf = dict(engine='parallel') return taskflow.engines.load(flow, flow_detail=flow_detail, - engine_conf=engine_conf, + engine='parallel', backend=self.backend, executor=executor) @@ -189,8 +188,7 @@ class ParallelEngineWithEventletTest(SuspendFlowTest, def _make_engine(self, flow, flow_detail=None, executor=None): if executor is None: executor = eu.GreenExecutor() - engine_conf = dict(engine='parallel') return taskflow.engines.load(flow, flow_detail=flow_detail, - engine_conf=engine_conf, + engine='parallel', backend=self.backend, executor=executor) From 7ca631356efd943bf8e246a6a907653a70a35771 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 2 Sep 2014 12:36:52 -0700 Subject: [PATCH 075/240] Use and verify event and latch wait() return using timeouts Instead of blocking up the whole test suite when a latch or event was not decremented to its desired value (or not set for an event) we should use a reasonably high value that we use when waiting for those actions to occur and verify that when those wait() functions return that we have reached the desired state and if not either raise an exception or stop further testing. Fixes bug 1363739 Change-Id: I8b40282ac2db9cabd48b0b65c8a2a49610d77c4f --- taskflow/conductors/single_threaded.py | 7 +++--- taskflow/engines/worker_based/proxy.py | 4 ++-- .../tests/unit/conductor/test_conductor.py | 19 ++++++++-------- taskflow/tests/unit/jobs/base.py | 22 ++++++++++++++----- taskflow/tests/unit/test_utils_lock_utils.py | 6 +++-- .../tests/unit/worker_based/test_executor.py | 18 +++++++-------- .../unit/worker_based/test_message_pump.py | 13 +++++------ taskflow/tests/utils.py | 18 +++++++-------- taskflow/types/timing.py | 6 ++--- taskflow/utils/threading_utils.py | 20 +++++++++++++++++ 10 files changed, 81 insertions(+), 52 deletions(-) diff --git a/taskflow/conductors/single_threaded.py b/taskflow/conductors/single_threaded.py index 23994e79..eef64836 100644 --- a/taskflow/conductors/single_threaded.py +++ b/taskflow/conductors/single_threaded.py @@ -13,7 +13,6 @@ # under the License. import logging -import threading import six @@ -23,6 +22,7 @@ from taskflow.listeners import logging as logging_listener from taskflow.types import timing as tt from taskflow.utils import async_utils from taskflow.utils import lock_utils +from taskflow.utils import threading_utils LOG = logging.getLogger(__name__) WAIT_TIMEOUT = 0.5 @@ -64,7 +64,7 @@ class SingleThreadedConductor(base.Conductor): self._wait_timeout = wait_timeout else: raise ValueError("Invalid timeout literal: %s" % (wait_timeout)) - self._dead = threading.Event() + self._dead = threading_utils.Event() @lock_utils.locked def stop(self, timeout=None): @@ -81,8 +81,7 @@ class SingleThreadedConductor(base.Conductor): be honored in the future) and False will be returned indicating this. """ self._wait_timeout.interrupt() - self._dead.wait(timeout) - return self._dead.is_set() + return self._dead.wait(timeout) @property def dispatching(self): diff --git a/taskflow/engines/worker_based/proxy.py b/taskflow/engines/worker_based/proxy.py index c51dd164..e0f9ce7d 100644 --- a/taskflow/engines/worker_based/proxy.py +++ b/taskflow/engines/worker_based/proxy.py @@ -16,13 +16,13 @@ import logging import socket -import threading import kombu import six from taskflow.engines.worker_based import dispatcher from taskflow.utils import misc +from taskflow.utils import threading_utils LOG = logging.getLogger(__name__) @@ -39,7 +39,7 @@ class Proxy(object): self._topic = topic self._exchange_name = exchange_name self._on_wait = on_wait - self._running = threading.Event() + self._running = threading_utils.Event() self._dispatcher = dispatcher.TypeDispatcher(type_handlers) self._dispatcher.add_requeue_filter( # NOTE(skudriashev): Process all incoming messages only if proxy is diff --git a/taskflow/tests/unit/conductor/test_conductor.py b/taskflow/tests/unit/conductor/test_conductor.py index b43ba035..2d254a30 100644 --- a/taskflow/tests/unit/conductor/test_conductor.py +++ b/taskflow/tests/unit/conductor/test_conductor.py @@ -30,6 +30,7 @@ from taskflow import test from taskflow.tests import utils as test_utils from taskflow.utils import misc from taskflow.utils import persistence_utils as pu +from taskflow.utils import threading_utils @contextlib.contextmanager @@ -88,14 +89,15 @@ class SingleThreadedConductorTest(test_utils.EngineTestBase, test.TestCase): with close_many(components.conductor, components.client): t = make_thread(components.conductor) t.start() - self.assertTrue(components.conductor.stop(0.5)) + self.assertTrue( + components.conductor.stop(test_utils.WAIT_TIMEOUT)) self.assertFalse(components.conductor.dispatching) t.join() def test_run(self): components = self.make_components() components.conductor.connect() - consumed_event = threading.Event() + consumed_event = threading_utils.Event() def on_consume(state, details): consumed_event.set() @@ -110,9 +112,8 @@ class SingleThreadedConductorTest(test_utils.EngineTestBase, test.TestCase): backend=components.persistence) components.board.post('poke', lb, details={'flow_uuid': fd.uuid}) - consumed_event.wait(1.0) - self.assertTrue(consumed_event.is_set()) - self.assertTrue(components.conductor.stop(1.0)) + self.assertTrue(consumed_event.wait(test_utils.WAIT_TIMEOUT)) + self.assertTrue(components.conductor.stop(test_utils.WAIT_TIMEOUT)) self.assertFalse(components.conductor.dispatching) persistence = components.persistence @@ -125,8 +126,7 @@ class SingleThreadedConductorTest(test_utils.EngineTestBase, test.TestCase): def test_fail_run(self): components = self.make_components() components.conductor.connect() - - consumed_event = threading.Event() + consumed_event = threading_utils.Event() def on_consume(state, details): consumed_event.set() @@ -141,9 +141,8 @@ class SingleThreadedConductorTest(test_utils.EngineTestBase, test.TestCase): backend=components.persistence) components.board.post('poke', lb, details={'flow_uuid': fd.uuid}) - consumed_event.wait(1.0) - self.assertTrue(consumed_event.is_set()) - self.assertTrue(components.conductor.stop(1.0)) + self.assertTrue(consumed_event.wait(test_utils.WAIT_TIMEOUT)) + self.assertTrue(components.conductor.stop(test_utils.WAIT_TIMEOUT)) self.assertFalse(components.conductor.dispatching) persistence = components.persistence diff --git a/taskflow/tests/unit/jobs/base.py b/taskflow/tests/unit/jobs/base.py index f8a2687e..e0a20a04 100644 --- a/taskflow/tests/unit/jobs/base.py +++ b/taskflow/tests/unit/jobs/base.py @@ -25,8 +25,10 @@ from taskflow.openstack.common import uuidutils from taskflow.persistence.backends import impl_dir from taskflow import states from taskflow.test import mock +from taskflow.tests import utils as test_utils from taskflow.utils import misc from taskflow.utils import persistence_utils as p_utils +from taskflow.utils import threading_utils FLUSH_PATH_TPL = '/taskflow/flush-test/%s' @@ -52,8 +54,8 @@ def flush(client, path=None): # before this context manager exits. if not path: path = FLUSH_PATH_TPL % uuidutils.generate_uuid() - created = threading.Event() - deleted = threading.Event() + created = threading_utils.Event() + deleted = threading_utils.Event() def on_created(data, stat): if stat is not None: @@ -67,13 +69,19 @@ def flush(client, path=None): watchers.DataWatch(client, path, func=on_created) client.create(path, makepath=True) - created.wait() + if not created.wait(test_utils.WAIT_TIMEOUT): + raise RuntimeError("Could not receive creation of %s in" + " the alloted timeout of %s seconds" + % (path, test_utils.WAIT_TIMEOUT)) try: yield finally: watchers.DataWatch(client, path, func=on_deleted) client.delete(path, recursive=True) - deleted.wait() + if not deleted.wait(test_utils.WAIT_TIMEOUT): + raise RuntimeError("Could not receive deletion of %s in" + " the alloted timeout of %s seconds" + % (path, test_utils.WAIT_TIMEOUT)) class BoardTestMixin(object): @@ -119,11 +127,13 @@ class BoardTestMixin(object): self.assertRaises(excp.NotFound, self.board.wait, timeout=0.1) def test_wait_arrival(self): - ev = threading.Event() + ev = threading_utils.Event() jobs = [] def poster(wait_post=0.2): - ev.wait() # wait until the waiter is active + if not ev.wait(test_utils.WAIT_TIMEOUT): + raise RuntimeError("Waiter did not appear ready" + " in %s seconds" % test_utils.WAIT_TIMEOUT) time.sleep(wait_post) self.board.post('test', p_utils.temporary_log_book()) diff --git a/taskflow/tests/unit/test_utils_lock_utils.py b/taskflow/tests/unit/test_utils_lock_utils.py index 066b17a2..c7fe09f1 100644 --- a/taskflow/tests/unit/test_utils_lock_utils.py +++ b/taskflow/tests/unit/test_utils_lock_utils.py @@ -22,7 +22,9 @@ from concurrent import futures import mock from taskflow import test +from taskflow.tests import utils as test_utils from taskflow.utils import lock_utils +from taskflow.utils import threading_utils # NOTE(harlowja): Sleep a little so time.time() can not be the same (which will # cause false positives when our overlap detection code runs). If there are @@ -353,7 +355,7 @@ class ReadWriteLockTest(test.TestCase): def test_double_reader_writer(self): lock = lock_utils.ReaderWriterLock() activated = collections.deque() - active = threading.Event() + active = threading_utils.Event() def double_reader(): with lock.read_lock(): @@ -369,7 +371,7 @@ class ReadWriteLockTest(test.TestCase): reader = threading.Thread(target=double_reader) reader.start() - active.wait() + self.assertTrue(active.wait(test_utils.WAIT_TIMEOUT)) writer = threading.Thread(target=happy_writer) writer.start() diff --git a/taskflow/tests/unit/worker_based/test_executor.py b/taskflow/tests/unit/worker_based/test_executor.py index 59681bb2..3e494e88 100644 --- a/taskflow/tests/unit/worker_based/test_executor.py +++ b/taskflow/tests/unit/worker_based/test_executor.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -import threading import time from concurrent import futures @@ -24,15 +23,16 @@ from taskflow.engines.worker_based import executor from taskflow.engines.worker_based import protocol as pr from taskflow import test from taskflow.test import mock -from taskflow.tests import utils +from taskflow.tests import utils as test_utils from taskflow.utils import misc +from taskflow.utils import threading_utils class TestWorkerTaskExecutor(test.MockTestCase): def setUp(self): super(TestWorkerTaskExecutor, self).setUp() - self.task = utils.DummyTask() + self.task = test_utils.DummyTask() self.task_uuid = 'task-uuid' self.task_args = {'a': 'a'} self.task_result = 'task-result' @@ -42,7 +42,7 @@ class TestWorkerTaskExecutor(test.MockTestCase): self.executor_uuid = 'executor-uuid' self.executor_exchange = 'executor-exchange' self.executor_topic = 'test-topic1' - self.proxy_started_event = threading.Event() + self.proxy_started_event = threading_utils.Event() # patch classes self.proxy_mock, self.proxy_inst_mock = self.patchClass( @@ -121,7 +121,7 @@ class TestWorkerTaskExecutor(test.MockTestCase): self.assertEqual(len(ex._requests_cache), 0) expected_calls = [ mock.call.transition_and_log_error(pr.FAILURE, logger=mock.ANY), - mock.call.set_result(result=utils.FailureMatcher(failure)) + mock.call.set_result(result=test_utils.FailureMatcher(failure)) ] self.assertEqual(expected_calls, self.request_inst_mock.mock_calls) @@ -303,7 +303,7 @@ class TestWorkerTaskExecutor(test.MockTestCase): ex.start() # make sure proxy thread started - self.proxy_started_event.wait() + self.assertTrue(self.proxy_started_event.wait(test_utils.WAIT_TIMEOUT)) # stop executor ex.stop() @@ -319,7 +319,7 @@ class TestWorkerTaskExecutor(test.MockTestCase): ex.start() # make sure proxy thread started - self.proxy_started_event.wait() + self.assertTrue(self.proxy_started_event.wait(test_utils.WAIT_TIMEOUT)) # start executor again ex.start() @@ -362,14 +362,14 @@ class TestWorkerTaskExecutor(test.MockTestCase): ex.start() # make sure thread started - self.proxy_started_event.wait() + self.assertTrue(self.proxy_started_event.wait(test_utils.WAIT_TIMEOUT)) # restart executor ex.stop() ex.start() # make sure thread started - self.proxy_started_event.wait() + self.assertTrue(self.proxy_started_event.wait(test_utils.WAIT_TIMEOUT)) # stop executor ex.stop() diff --git a/taskflow/tests/unit/worker_based/test_message_pump.py b/taskflow/tests/unit/worker_based/test_message_pump.py index 008ad72c..1fc946ed 100644 --- a/taskflow/tests/unit/worker_based/test_message_pump.py +++ b/taskflow/tests/unit/worker_based/test_message_pump.py @@ -23,15 +23,15 @@ from taskflow import test from taskflow.test import mock from taskflow.tests import utils as test_utils from taskflow.types import latch +from taskflow.utils import threading_utils TEST_EXCHANGE, TEST_TOPIC = ('test-exchange', 'test-topic') -BARRIER_WAIT_TIMEOUT = 1.0 POLLING_INTERVAL = 0.01 class TestMessagePump(test.TestCase): def test_notify(self): - barrier = threading.Event() + barrier = threading_utils.Event() on_notify = mock.MagicMock() on_notify.side_effect = lambda *args, **kwargs: barrier.set() @@ -49,8 +49,7 @@ class TestMessagePump(test.TestCase): p.wait() p.publish(pr.Notify(), TEST_TOPIC) - barrier.wait(BARRIER_WAIT_TIMEOUT) - self.assertTrue(barrier.is_set()) + self.assertTrue(barrier.wait(test_utils.WAIT_TIMEOUT)) p.stop() t.join() @@ -58,7 +57,7 @@ class TestMessagePump(test.TestCase): on_notify.assert_called_with({}, mock.ANY) def test_response(self): - barrier = threading.Event() + barrier = threading_utils.Event() on_response = mock.MagicMock() on_response.side_effect = lambda *args, **kwargs: barrier.set() @@ -77,7 +76,7 @@ class TestMessagePump(test.TestCase): resp = pr.Response(pr.RUNNING) p.publish(resp, TEST_TOPIC) - barrier.wait(BARRIER_WAIT_TIMEOUT) + self.assertTrue(barrier.wait(test_utils.WAIT_TIMEOUT)) self.assertTrue(barrier.is_set()) p.stop() t.join() @@ -126,7 +125,7 @@ class TestMessagePump(test.TestCase): uuidutils.generate_uuid(), pr.EXECUTE, [], None, None), TEST_TOPIC) - barrier.wait(BARRIER_WAIT_TIMEOUT) + self.assertTrue(barrier.wait(test_utils.WAIT_TIMEOUT)) self.assertEqual(0, barrier.needed) p.stop() t.join() diff --git a/taskflow/tests/utils.py b/taskflow/tests/utils.py index 1662b286..d01f91a3 100644 --- a/taskflow/tests/utils.py +++ b/taskflow/tests/utils.py @@ -16,7 +16,6 @@ import contextlib import string -import threading import six @@ -26,15 +25,18 @@ from taskflow import retry from taskflow import task from taskflow.utils import kazoo_utils from taskflow.utils import misc +from taskflow.utils import threading_utils ARGS_KEY = '__args__' KWARGS_KEY = '__kwargs__' ORDER_KEY = '__order__' - ZK_TEST_CONFIG = { 'timeout': 1.0, 'hosts': ["localhost:2181"], } +# If latches/events take longer than this to become empty/set, something is +# usually wrong and should be debugged instead of deadlocking... +WAIT_TIMEOUT = 300 @contextlib.contextmanager @@ -342,16 +344,14 @@ class WaitForOneFromTask(SaveOrderTask): self.wait_states = [wait_states] else: self.wait_states = wait_states - self.event = threading.Event() + self.event = threading_utils.Event() def execute(self): - # NOTE(imelnikov): if test was not complete within - # 5 minutes, something is terribly wrong - self.event.wait(300) - if not self.event.is_set(): - raise RuntimeError('Timeout occurred while waiting ' + if not self.event.wait(WAIT_TIMEOUT): + raise RuntimeError('%s second timeout occurred while waiting ' 'for %s to change state to %s' - % (self.wait_for, self.wait_states)) + % (WAIT_TIMEOUT, self.wait_for, + self.wait_states)) return super(WaitForOneFromTask, self).execute() def callback(self, state, details): diff --git a/taskflow/types/timing.py b/taskflow/types/timing.py index 6e2c46c8..decada4a 100644 --- a/taskflow/types/timing.py +++ b/taskflow/types/timing.py @@ -14,10 +14,10 @@ # License for the specific language governing permissions and limitations # under the License. -import threading - from oslo.utils import timeutils +from taskflow.utils import threading_utils + class Timeout(object): """An object which represents a timeout. @@ -29,7 +29,7 @@ class Timeout(object): if timeout < 0: raise ValueError("Timeout must be >= 0 and not %s" % (timeout)) self._timeout = timeout - self._event = threading.Event() + self._event = threading_utils.Event() def interrupt(self): self._event.set() diff --git a/taskflow/utils/threading_utils.py b/taskflow/utils/threading_utils.py index 2af17023..b3749bca 100644 --- a/taskflow/utils/threading_utils.py +++ b/taskflow/utils/threading_utils.py @@ -15,11 +15,31 @@ # under the License. import multiprocessing +import sys import threading from six.moves import _thread +if sys.version_info[0:2] == (2, 6): + # This didn't return that was/wasn't set in 2.6, since we actually care + # whether it did or didn't add that feature by taking the code from 2.7 + # that added this functionality... + # + # TODO(harlowja): remove when we can drop 2.6 support. + class Event(threading._Event): + def wait(self, timeout=None): + self.__cond.acquire() + try: + if not self.__flag: + self.__cond.wait(timeout) + return self.__flag + finally: + self.__cond.release() +else: + Event = threading.Event + + def get_ident(): """Return the 'thread identifier' of the current thread.""" return _thread.get_ident() From ac77b4dce98a39bdfba6df7b18006ab447296e7a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 18 Oct 2014 18:22:38 -0700 Subject: [PATCH 076/240] Bump the deprecation version number These will start to emit deprecation warning messages in the next version/release (0.6) and not the current version (0.5) so we should update the messaging to reflect this. Change-Id: I8823741847b42056690629fa12c612c8d1ef2cff --- taskflow/utils/misc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index a4619881..57df225b 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -392,11 +392,11 @@ def ensure_tree(path): Failure = deprecation.moved_class(failure.Failure, 'Failure', __name__, - version="0.5", removal_version="?") + version="0.6", removal_version="?") Notifier = deprecation.moved_class(notifier.Notifier, 'Notifier', __name__, - version="0.5", removal_version="?") + version="0.6", removal_version="?") @contextlib.contextmanager From b014fc7d48969bd6812a11a5a0342c9324108876 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 23 Aug 2014 20:09:10 -0700 Subject: [PATCH 077/240] Add a futures type that can unify our future functionality Move the currently existing green future executor and associated code to a new futures types module so that it can be accessed from this new location (TODO: deprecate the old location and link the old to the new for one release so that we can remove the old link in N + 1 release). This unifies the API that the existing pool (thread or process) future executors and the green thread pool future executor, and the newly added synchronous executor (replacing the previous `make_completed_future` function) provide so there usage is as seamless as possible. Part of blueprint top-level-types Change-Id: Ie5500eaa7f4425edb604b2dd13a15f82909a673b --- doc/source/types.rst | 5 + taskflow/engines/action_engine/executor.py | 17 +- .../engines/action_engine/retry_action.py | 62 +++-- taskflow/examples/resume_vm_boot.py | 7 +- .../persistence/backends/impl_sqlalchemy.py | 4 +- taskflow/tests/unit/test_engines.py | 8 +- taskflow/tests/unit/test_futures.py | 222 ++++++++++++++++++ taskflow/tests/unit/test_green_executor.py | 131 ----------- taskflow/tests/unit/test_suspend_flow.py | 7 +- taskflow/tests/unit/test_utils_async_utils.py | 45 ++-- .../eventlet_utils.py => types/futures.py} | 178 +++++++------- taskflow/utils/async_utils.py | 82 ++++++- 12 files changed, 480 insertions(+), 288 deletions(-) create mode 100644 taskflow/tests/unit/test_futures.py delete mode 100644 taskflow/tests/unit/test_green_executor.py rename taskflow/{utils/eventlet_utils.py => types/futures.py} (51%) diff --git a/doc/source/types.rst b/doc/source/types.rst index c628c1ce..fb9580af 100644 --- a/doc/source/types.rst +++ b/doc/source/types.rst @@ -17,6 +17,11 @@ FSM .. automodule:: taskflow.types.fsm +Futures +======= + +.. automodule:: taskflow.types.futures + Graph ===== diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index 40a671eb..83da3b6b 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -16,10 +16,10 @@ import abc -from concurrent import futures import six from taskflow import task as _task +from taskflow.types import futures from taskflow.utils import async_utils from taskflow.utils import misc from taskflow.utils import threading_utils @@ -94,19 +94,20 @@ class TaskExecutorBase(object): class SerialTaskExecutor(TaskExecutorBase): """Execute task one after another.""" + def __init__(self): + self._executor = futures.SynchronousExecutor() + def execute_task(self, task, task_uuid, arguments, progress_callback=None): - return async_utils.make_completed_future( - _execute_task(task, arguments, progress_callback)) + return self._executor.submit(_execute_task, task, arguments, + progress_callback) def revert_task(self, task, task_uuid, arguments, result, failures, progress_callback=None): - return async_utils.make_completed_future( - _revert_task(task, arguments, result, - failures, progress_callback)) + return self._executor.submit(_revert_task, task, arguments, result, + failures, progress_callback) def wait_for_any(self, fs, timeout=None): - # NOTE(imelnikov): this executor returns only done futures. - return (fs, set()) + return async_utils.wait_for_any(fs, timeout) class ParallelTaskExecutor(TaskExecutorBase): diff --git a/taskflow/engines/action_engine/retry_action.py b/taskflow/engines/action_engine/retry_action.py index e4df5afa..3bf6f491 100644 --- a/taskflow/engines/action_engine/retry_action.py +++ b/taskflow/engines/action_engine/retry_action.py @@ -18,7 +18,7 @@ import logging from taskflow.engines.action_engine import executor as ex from taskflow import states -from taskflow.utils import async_utils +from taskflow.types import futures from taskflow.utils import misc LOG = logging.getLogger(__name__) @@ -31,6 +31,7 @@ class RetryAction(object): self._storage = storage self._notifier = notifier self._walker_factory = walker_factory + self._executor = futures.SynchronousExecutor() def _get_retry_args(self, retry): scope_walker = self._walker_factory(retry) @@ -59,29 +60,50 @@ class RetryAction(object): self._notifier.notify(state, details) def execute(self, retry): + + def _execute_retry(kwargs): + try: + result = retry.execute(**kwargs) + except Exception: + result = misc.Failure() + return (retry, ex.EXECUTED, result) + + def _on_done_callback(fut): + result = fut.result()[-1] + if isinstance(result, misc.Failure): + self.change_state(retry, states.FAILURE, result=result) + else: + self.change_state(retry, states.SUCCESS, result=result) + self.change_state(retry, states.RUNNING) - kwargs = self._get_retry_args(retry) - try: - result = retry.execute(**kwargs) - except Exception: - result = misc.Failure() - self.change_state(retry, states.FAILURE, result=result) - else: - self.change_state(retry, states.SUCCESS, result=result) - return async_utils.make_completed_future((retry, ex.EXECUTED, result)) + fut = self._executor.submit(_execute_retry, + self._get_retry_args(retry)) + fut.add_done_callback(_on_done_callback) + return fut def revert(self, retry): + + def _execute_retry(kwargs, failures): + kwargs['flow_failures'] = failures + try: + result = retry.revert(**kwargs) + except Exception: + result = misc.Failure() + return (retry, ex.REVERTED, result) + + def _on_done_callback(fut): + result = fut.result()[-1] + if isinstance(result, misc.Failure): + self.change_state(retry, states.FAILURE) + else: + self.change_state(retry, states.REVERTED) + self.change_state(retry, states.REVERTING) - kwargs = self._get_retry_args(retry) - kwargs['flow_failures'] = self._storage.get_failures() - try: - result = retry.revert(**kwargs) - except Exception: - result = misc.Failure() - self.change_state(retry, states.FAILURE) - else: - self.change_state(retry, states.REVERTED) - return async_utils.make_completed_future((retry, ex.REVERTED, result)) + fut = self._executor.submit(_execute_retry, + self._get_retry_args(retry), + self._storage.get_failures()) + fut.add_done_callback(_on_done_callback) + return fut def on_failure(self, retry, atom, last_failure): self._storage.save_retry_failure(retry.name, atom.name, last_failure) diff --git a/taskflow/examples/resume_vm_boot.py b/taskflow/examples/resume_vm_boot.py index 203cb882..514f3336 100644 --- a/taskflow/examples/resume_vm_boot.py +++ b/taskflow/examples/resume_vm_boot.py @@ -37,7 +37,8 @@ from taskflow.openstack.common import uuidutils from taskflow.patterns import graph_flow as gf from taskflow.patterns import linear_flow as lf from taskflow import task -from taskflow.utils import eventlet_utils as e_utils +from taskflow.types import futures +from taskflow.utils import async_utils from taskflow.utils import persistence_utils as p_utils import example_utils as eu # noqa @@ -236,8 +237,8 @@ with eu.get_backend() as backend: # Set up how we want our engine to run, serial, parallel... executor = None - if e_utils.EVENTLET_AVAILABLE: - executor = e_utils.GreenExecutor(5) + if async_utils.EVENTLET_AVAILABLE: + executor = futures.GreenThreadPoolExecutor(5) # Create/fetch a logbook that will track the workflows work. book = None diff --git a/taskflow/persistence/backends/impl_sqlalchemy.py b/taskflow/persistence/backends/impl_sqlalchemy.py index 587d4d25..29ab8c97 100644 --- a/taskflow/persistence/backends/impl_sqlalchemy.py +++ b/taskflow/persistence/backends/impl_sqlalchemy.py @@ -37,7 +37,7 @@ from taskflow.persistence.backends import base from taskflow.persistence.backends.sqlalchemy import migration from taskflow.persistence.backends.sqlalchemy import models from taskflow.persistence import logbook -from taskflow.utils import eventlet_utils +from taskflow.utils import async_utils from taskflow.utils import misc @@ -249,7 +249,7 @@ class SQLAlchemyBackend(base.Backend): engine_args.update(conf.pop('engine_args', {})) engine = sa.create_engine(sql_connection, **engine_args) checkin_yield = conf.pop('checkin_yield', - eventlet_utils.EVENTLET_AVAILABLE) + async_utils.EVENTLET_AVAILABLE) if _as_bool(checkin_yield): sa.event.listen(engine, 'checkin', _thread_yield) if 'mysql' in e_url.drivername: diff --git a/taskflow/tests/unit/test_engines.py b/taskflow/tests/unit/test_engines.py index 4ce291d1..0823002d 100644 --- a/taskflow/tests/unit/test_engines.py +++ b/taskflow/tests/unit/test_engines.py @@ -17,7 +17,6 @@ import contextlib import threading -from concurrent import futures import testtools import taskflow.engines @@ -33,8 +32,9 @@ from taskflow import states from taskflow import task from taskflow import test from taskflow.tests import utils +from taskflow.types import futures from taskflow.types import graph as gr -from taskflow.utils import eventlet_utils as eu +from taskflow.utils import async_utils as au from taskflow.utils import misc from taskflow.utils import persistence_utils as p_utils @@ -590,7 +590,7 @@ class MultiThreadedEngineTest(EngineTaskTest, executor.shutdown(wait=True) -@testtools.skipIf(not eu.EVENTLET_AVAILABLE, 'eventlet is not available') +@testtools.skipIf(not au.EVENTLET_AVAILABLE, 'eventlet is not available') class ParallelEngineWithEventletTest(EngineTaskTest, EngineLinearFlowTest, EngineParallelFlowTest, @@ -601,7 +601,7 @@ class ParallelEngineWithEventletTest(EngineTaskTest, def _make_engine(self, flow, flow_detail=None, executor=None): if executor is None: - executor = eu.GreenExecutor() + executor = futures.GreenThreadPoolExecutor() return taskflow.engines.load(flow, flow_detail=flow_detail, backend=self.backend, engine='parallel', executor=executor) diff --git a/taskflow/tests/unit/test_futures.py b/taskflow/tests/unit/test_futures.py new file mode 100644 index 00000000..576b5eee --- /dev/null +++ b/taskflow/tests/unit/test_futures.py @@ -0,0 +1,222 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2013 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +import functools +import threading +import time + +import testtools + +from taskflow import test +from taskflow.types import futures + +try: + from eventlet.green import threading as greenthreading + from eventlet.green import time as greentime + EVENTLET_AVAILABLE = True +except ImportError: + EVENTLET_AVAILABLE = False + + +def _noop(): + pass + + +def _blowup(): + raise IOError("Broke!") + + +def _return_given(given): + return given + + +def _return_one(): + return 1 + + +def _double(x): + return x * 2 + + +class _SimpleFuturesTestMixin(object): + # This exists to test basic functionality, mainly required to test the + # process executor which has a very restricted set of things it can + # execute (no lambda functions, no instance methods...) + def _make_executor(self, max_workers): + raise NotImplementedError("Not implemented") + + def test_invalid_workers(self): + self.assertRaises(ValueError, self._make_executor, -1) + self.assertRaises(ValueError, self._make_executor, 0) + + def test_exception_transfer(self): + with self._make_executor(2) as e: + f = e.submit(_blowup) + self.assertRaises(IOError, f.result) + + def test_accumulator(self): + created = [] + with self._make_executor(5) as e: + for _i in range(0, 10): + created.append(e.submit(_return_one)) + results = [f.result() for f in created] + self.assertEqual(10, sum(results)) + + def test_map(self): + count = [i for i in range(0, 100)] + with self._make_executor(5) as e: + results = list(e.map(_double, count)) + initial = sum(count) + self.assertEqual(2 * initial, sum(results)) + + def test_alive(self): + e = self._make_executor(1) + self.assertTrue(e.alive) + e.shutdown() + self.assertFalse(e.alive) + with self._make_executor(1) as e2: + self.assertTrue(e2.alive) + self.assertFalse(e2.alive) + + +class _FuturesTestMixin(_SimpleFuturesTestMixin): + def _delay(self, secs): + raise NotImplementedError("Not implemented") + + def _make_lock(self): + raise NotImplementedError("Not implemented") + + def _make_funcs(self, called, amount): + mutator = self._make_lock() + + def store_call(ident): + with mutator: + called[ident] += 1 + + for i in range(0, amount): + yield functools.partial(store_call, ident=i) + + def test_func_calls(self): + called = collections.defaultdict(int) + + with self._make_executor(2) as e: + for f in self._make_funcs(called, 2): + e.submit(f) + + self.assertEqual(1, called[0]) + self.assertEqual(1, called[1]) + + def test_result_callback(self): + called = collections.defaultdict(int) + mutator = self._make_lock() + + def callback(future): + with mutator: + called[future] += 1 + + funcs = list(self._make_funcs(called, 1)) + with self._make_executor(2) as e: + for func in funcs: + f = e.submit(func) + f.add_done_callback(callback) + + self.assertEqual(2, len(called)) + + def test_result_transfer(self): + create_am = 50 + with self._make_executor(2) as e: + fs = [] + for i in range(0, create_am): + fs.append(e.submit(functools.partial(_return_given, i))) + self.assertEqual(create_am, len(fs)) + for i in range(0, create_am): + result = fs[i].result() + self.assertEqual(i, result) + + def test_called_restricted_size(self): + called = collections.defaultdict(int) + + with self._make_executor(1) as e: + for f in self._make_funcs(called, 100): + e.submit(f) + + self.assertFalse(e.alive) + self.assertEqual(100, len(called)) + + +class ThreadPoolExecutorTest(test.TestCase, _FuturesTestMixin): + def _make_executor(self, max_workers): + return futures.ThreadPoolExecutor(max_workers=max_workers) + + def _delay(self, secs): + time.sleep(secs) + + def _make_lock(self): + return threading.Lock() + + +class ProcessPoolExecutorTest(test.TestCase, _SimpleFuturesTestMixin): + def _make_executor(self, max_workers): + return futures.ProcessPoolExecutor(max_workers=max_workers) + + +class SynchronousExecutorTest(test.TestCase, _FuturesTestMixin): + def _make_executor(self, max_workers): + return futures.SynchronousExecutor() + + def _delay(self, secs): + time.sleep(secs) + + def _make_lock(self): + return threading.Lock() + + def test_invalid_workers(self): + pass + + +@testtools.skipIf(not EVENTLET_AVAILABLE, 'eventlet is not available') +class GreenThreadPoolExecutorTest(test.TestCase, _FuturesTestMixin): + def _make_executor(self, max_workers): + return futures.GreenThreadPoolExecutor(max_workers=max_workers) + + def _delay(self, secs): + greentime.sleep(secs) + + def _make_lock(self): + return greenthreading.Lock() + + def test_cancellation(self): + called = collections.defaultdict(int) + + fs = [] + with self._make_executor(2) as e: + for func in self._make_funcs(called, 2): + fs.append(e.submit(func)) + # Greenthreads don't start executing until we wait for them + # to, since nothing here does IO, this will work out correctly. + # + # If something here did a blocking call, then eventlet could swap + # one of the executors threads in, but nothing in this test does. + for f in fs: + self.assertFalse(f.running()) + f.cancel() + + self.assertEqual(0, len(called)) + self.assertEqual(2, len(fs)) + for f in fs: + self.assertTrue(f.cancelled()) + self.assertTrue(f.done()) diff --git a/taskflow/tests/unit/test_green_executor.py b/taskflow/tests/unit/test_green_executor.py deleted file mode 100644 index eae523dc..00000000 --- a/taskflow/tests/unit/test_green_executor.py +++ /dev/null @@ -1,131 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (C) 2013 Yahoo! Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import collections -import functools - -import testtools - -from taskflow import test -from taskflow.utils import eventlet_utils as eu - - -@testtools.skipIf(not eu.EVENTLET_AVAILABLE, 'eventlet is not available') -class GreenExecutorTest(test.TestCase): - def make_funcs(self, called, amount): - - def store_call(name): - called[name] += 1 - - for i in range(0, amount): - yield functools.partial(store_call, name=i) - - def test_func_calls(self): - called = collections.defaultdict(int) - - with eu.GreenExecutor(2) as e: - for f in self.make_funcs(called, 2): - e.submit(f) - - self.assertEqual(1, called[0]) - self.assertEqual(1, called[1]) - - def test_no_construction(self): - self.assertRaises(ValueError, eu.GreenExecutor, 0) - self.assertRaises(ValueError, eu.GreenExecutor, -1) - self.assertRaises(ValueError, eu.GreenExecutor, "-1") - - def test_result_callback(self): - called = collections.defaultdict(int) - - def callback(future): - called[future] += 1 - - funcs = list(self.make_funcs(called, 1)) - with eu.GreenExecutor(2) as e: - for func in funcs: - f = e.submit(func) - f.add_done_callback(callback) - - self.assertEqual(2, len(called)) - - def test_exception_transfer(self): - - def blowup(): - raise IOError("Broke!") - - with eu.GreenExecutor(2) as e: - f = e.submit(blowup) - - self.assertRaises(IOError, f.result) - - def test_result_transfer(self): - - def return_given(given): - return given - - create_am = 50 - with eu.GreenExecutor(2) as e: - fs = [] - for i in range(0, create_am): - fs.append(e.submit(functools.partial(return_given, i))) - - self.assertEqual(create_am, len(fs)) - for i in range(0, create_am): - result = fs[i].result() - self.assertEqual(i, result) - - def test_called_restricted_size(self): - called = collections.defaultdict(int) - - with eu.GreenExecutor(1) as e: - for f in self.make_funcs(called, 100): - e.submit(f) - self.assertEqual(99, e.amount_delayed) - - self.assertFalse(e.alive) - self.assertEqual(100, len(called)) - self.assertGreaterEqual(1, e.workers_created) - self.assertEqual(0, e.amount_delayed) - - def test_shutdown_twice(self): - e = eu.GreenExecutor(1) - self.assertTrue(e.alive) - e.shutdown() - self.assertFalse(e.alive) - e.shutdown() - self.assertFalse(e.alive) - - def test_func_cancellation(self): - called = collections.defaultdict(int) - - fs = [] - with eu.GreenExecutor(2) as e: - for func in self.make_funcs(called, 2): - fs.append(e.submit(func)) - # Greenthreads don't start executing until we wait for them - # to, since nothing here does IO, this will work out correctly. - # - # If something here did a blocking call, then eventlet could swap - # one of the executors threads in, but nothing in this test does. - for f in fs: - self.assertFalse(f.running()) - f.cancel() - - self.assertEqual(0, len(called)) - for f in fs: - self.assertTrue(f.cancelled()) - self.assertTrue(f.done()) diff --git a/taskflow/tests/unit/test_suspend_flow.py b/taskflow/tests/unit/test_suspend_flow.py index 7b9875c9..928f2bec 100644 --- a/taskflow/tests/unit/test_suspend_flow.py +++ b/taskflow/tests/unit/test_suspend_flow.py @@ -23,7 +23,8 @@ from taskflow.patterns import linear_flow as lf from taskflow import states from taskflow import test from taskflow.tests import utils -from taskflow.utils import eventlet_utils as eu +from taskflow.types import futures +from taskflow.utils import async_utils as au class SuspendingListener(lbase.ListenerBase): @@ -181,13 +182,13 @@ class MultiThreadedEngineTest(SuspendFlowTest, executor=executor) -@testtools.skipIf(not eu.EVENTLET_AVAILABLE, 'eventlet is not available') +@testtools.skipIf(not au.EVENTLET_AVAILABLE, 'eventlet is not available') class ParallelEngineWithEventletTest(SuspendFlowTest, test.TestCase): def _make_engine(self, flow, flow_detail=None, executor=None): if executor is None: - executor = eu.GreenExecutor() + executor = futures.GreenThreadPoolExecutor() return taskflow.engines.load(flow, flow_detail=flow_detail, engine='parallel', backend=self.backend, diff --git a/taskflow/tests/unit/test_utils_async_utils.py b/taskflow/tests/unit/test_utils_async_utils.py index 0abf4107..7bb033b8 100644 --- a/taskflow/tests/unit/test_utils_async_utils.py +++ b/taskflow/tests/unit/test_utils_async_utils.py @@ -14,12 +14,11 @@ # License for the specific language governing permissions and limitations # under the License. -from concurrent import futures import testtools from taskflow import test +from taskflow.types import futures from taskflow.utils import async_utils as au -from taskflow.utils import eventlet_utils as eu class WaitForAnyTestsMixin(object): @@ -29,7 +28,7 @@ class WaitForAnyTestsMixin(object): def foo(): pass - with self.executor_cls(2) as e: + with self._make_executor(2) as e: fs = [e.submit(foo), e.submit(foo)] # this test assumes that our foo will end within 10 seconds done, not_done = au.wait_for_any(fs, 10) @@ -53,34 +52,17 @@ class WaitForAnyTestsMixin(object): self.assertIs(done.pop(), f2) -@testtools.skipIf(not eu.EVENTLET_AVAILABLE, 'eventlet is not available') +@testtools.skipIf(not au.EVENTLET_AVAILABLE, 'eventlet is not available') class AsyncUtilsEventletTest(test.TestCase, WaitForAnyTestsMixin): - executor_cls = eu.GreenExecutor - is_green = True - - def test_add_result(self): - waiter = eu._GreenWaiter() - self.assertFalse(waiter.event.is_set()) - waiter.add_result(futures.Future()) - self.assertTrue(waiter.event.is_set()) - - def test_add_exception(self): - waiter = eu._GreenWaiter() - self.assertFalse(waiter.event.is_set()) - waiter.add_exception(futures.Future()) - self.assertTrue(waiter.event.is_set()) - - def test_add_cancelled(self): - waiter = eu._GreenWaiter() - self.assertFalse(waiter.event.is_set()) - waiter.add_cancelled(futures.Future()) - self.assertTrue(waiter.event.is_set()) + def _make_executor(self, max_workers): + return futures.GreenThreadPoolExecutor(max_workers=max_workers) class AsyncUtilsThreadedTest(test.TestCase, WaitForAnyTestsMixin): - executor_cls = futures.ThreadPoolExecutor + def _make_executor(self, max_workers): + return futures.ThreadPoolExecutor(max_workers=max_workers) class MakeCompletedFutureTest(test.TestCase): @@ -90,3 +72,16 @@ class MakeCompletedFutureTest(test.TestCase): future = au.make_completed_future(result) self.assertTrue(future.done()) self.assertIs(future.result(), result) + + def test_make_completed_future_exception(self): + result = IOError("broken") + future = au.make_completed_future(result, exception=True) + self.assertTrue(future.done()) + self.assertRaises(IOError, future.result) + self.assertIsNotNone(future.exception()) + + +class AsyncUtilsSynchronousTest(test.TestCase, + WaitForAnyTestsMixin): + def _make_executor(self, max_workers): + return futures.SynchronousExecutor() diff --git a/taskflow/utils/eventlet_utils.py b/taskflow/types/futures.py similarity index 51% rename from taskflow/utils/eventlet_utils.py rename to taskflow/types/futures.py index 335dd017..194730e5 100644 --- a/taskflow/utils/eventlet_utils.py +++ b/taskflow/types/futures.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2013 Yahoo! Inc. All Rights Reserved. +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -14,9 +14,9 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - -from concurrent import futures +from concurrent import futures as _futures +from concurrent.futures import process as _process +from concurrent.futures import thread as _thread try: from eventlet.green import threading as greenthreading @@ -27,15 +27,45 @@ try: except ImportError: EVENTLET_AVAILABLE = False +from taskflow.utils import threading_utils as tu -from taskflow.utils import lock_utils -LOG = logging.getLogger(__name__) +# NOTE(harlowja): Allows for simpler access to this type... +Future = _futures.Future -_DONE_STATES = frozenset([ - futures._base.CANCELLED_AND_NOTIFIED, - futures._base.FINISHED, -]) + +class ThreadPoolExecutor(_thread.ThreadPoolExecutor): + """Executor that uses a thread pool to execute calls asynchronously. + + See: https://docs.python.org/dev/library/concurrent.futures.html + """ + def __init__(self, max_workers=None): + if max_workers is None: + max_workers = tu.get_optimal_thread_count() + super(ThreadPoolExecutor, self).__init__(max_workers=max_workers) + if self._max_workers <= 0: + raise ValueError("Max workers must be greater than zero") + + @property + def alive(self): + return not self._shutdown + + +class ProcessPoolExecutor(_process.ProcessPoolExecutor): + """Executor that uses a process pool to execute calls asynchronously. + + See: https://docs.python.org/dev/library/concurrent.futures.html + """ + def __init__(self, max_workers=None): + if max_workers is None: + max_workers = tu.get_optimal_thread_count() + super(ProcessPoolExecutor, self).__init__(max_workers=max_workers) + if self._max_workers <= 0: + raise ValueError("Max workers must be greater than zero") + + @property + def alive(self): + return not self._shutdown_thread class _WorkItem(object): @@ -56,7 +86,36 @@ class _WorkItem(object): self.future.set_result(result) -class _Worker(object): +class SynchronousExecutor(_futures.Executor): + """Executor that uses the caller to execute calls synchronously. + + This provides an interface to a caller that looks like an executor but + will execute the calls inside the caller thread instead of executing it + in a external process/thread for when this type of functionality is + useful to provide... + """ + + def __init__(self): + self._shutoff = False + + @property + def alive(self): + return not self._shutoff + + def shutdown(self, wait=True): + self._shutoff = True + + def submit(self, fn, *args, **kwargs): + if self._shutoff: + raise RuntimeError('Can not schedule new futures' + ' after being shutdown') + f = Future() + runner = _WorkItem(f, fn, args, kwargs) + runner.run() + return f + + +class _GreenWorker(object): def __init__(self, executor, work, work_queue): self.executor = executor self.work = work @@ -82,7 +141,7 @@ class _Worker(object): self.work_queue.task_done() -class GreenFuture(futures.Future): +class GreenFuture(Future): def __init__(self): super(GreenFuture, self).__init__() assert EVENTLET_AVAILABLE, 'eventlet is needed to use a green future' @@ -95,96 +154,53 @@ class GreenFuture(futures.Future): self._condition = greenthreading.Condition() -class GreenExecutor(futures.Executor): - """A greenthread backed executor.""" +class GreenThreadPoolExecutor(_futures.Executor): + """Executor that uses a green thread pool to execute calls asynchronously. + + See: https://docs.python.org/dev/library/concurrent.futures.html + and http://eventlet.net/doc/modules/greenpool.html for information on + how this works. + """ def __init__(self, max_workers=1000): assert EVENTLET_AVAILABLE, 'eventlet is needed to use a green executor' - self._max_workers = int(max_workers) - if self._max_workers <= 0: - raise ValueError('Max workers must be greater than zero') + if max_workers <= 0: + raise ValueError("Max workers must be greater than zero") + self._max_workers = max_workers self._pool = greenpool.GreenPool(self._max_workers) self._delayed_work = greenqueue.Queue() self._shutdown_lock = greenthreading.Lock() self._shutdown = False - self._workers_created = 0 - - @property - def workers_created(self): - return self._workers_created - - @property - def amount_delayed(self): - return self._delayed_work.qsize() @property def alive(self): return not self._shutdown - @lock_utils.locked(lock='_shutdown_lock') def submit(self, fn, *args, **kwargs): - if self._shutdown: - raise RuntimeError('cannot schedule new futures after shutdown') - f = GreenFuture() - work = _WorkItem(f, fn, args, kwargs) - if not self._spin_up(work): - self._delayed_work.put(work) - return f + with self._shutdown_lock: + if self._shutdown: + raise RuntimeError('Can not schedule new futures' + ' after being shutdown') + f = GreenFuture() + work = _WorkItem(f, fn, args, kwargs) + if not self._spin_up(work): + self._delayed_work.put(work) + return f def _spin_up(self, work): alive = self._pool.running() + self._pool.waiting() if alive < self._max_workers: - self._pool.spawn_n(_Worker(self, work, self._delayed_work)) - self._workers_created += 1 + self._pool.spawn_n(_GreenWorker(self, work, self._delayed_work)) return True return False def shutdown(self, wait=True): with self._shutdown_lock: - self._shutdown = True - if wait: + if not self._shutdown: + self._shutdown = True + shutoff = True + else: + shutoff = False + if wait and shutoff: self._pool.waitall() self._delayed_work.join() - - -class _GreenWaiter(object): - """Provides the event that wait_for_any() blocks on.""" - def __init__(self): - self.event = greenthreading.Event() - - def add_result(self, future): - self.event.set() - - def add_exception(self, future): - self.event.set() - - def add_cancelled(self, future): - self.event.set() - - -def _partition_futures(fs): - """Partitions the input futures into done and not done lists.""" - done = set() - not_done = set() - for f in fs: - if f._state in _DONE_STATES: - done.add(f) - else: - not_done.add(f) - return (done, not_done) - - -def wait_for_any(fs, timeout=None): - assert EVENTLET_AVAILABLE, ('eventlet is needed to wait on green futures') - with futures._base._AcquireFutures(fs): - (done, not_done) = _partition_futures(fs) - if done: - return (done, not_done) - waiter = _GreenWaiter() - for f in fs: - f._waiters.append(waiter) - waiter.event.wait(timeout) - for f in fs: - f._waiters.remove(waiter) - with futures._base._AcquireFutures(fs): - return _partition_futures(fs) diff --git a/taskflow/utils/async_utils.py b/taskflow/utils/async_utils.py index 6d280dfe..b055a27b 100644 --- a/taskflow/utils/async_utils.py +++ b/taskflow/utils/async_utils.py @@ -14,9 +14,32 @@ # License for the specific language governing permissions and limitations # under the License. -from concurrent import futures +from concurrent import futures as _futures +from concurrent.futures import _base -from taskflow.utils import eventlet_utils as eu +try: + from eventlet.green import threading as greenthreading + EVENTLET_AVAILABLE = True +except ImportError: + EVENTLET_AVAILABLE = False + +from taskflow.types import futures + + +_DONE_STATES = frozenset([ + _base.CANCELLED_AND_NOTIFIED, + _base.FINISHED, +]) + + +def make_completed_future(result, exception=False): + """Make a future completed with a given result.""" + future = futures.Future() + if exception: + future.set_exception(result) + else: + future.set_result(result) + return future def wait_for_any(fs, timeout=None): @@ -29,10 +52,10 @@ def wait_for_any(fs, timeout=None): Returns pair (done futures, not done futures). """ - green_fs = sum(1 for f in fs if isinstance(f, eu.GreenFuture)) + green_fs = sum(1 for f in fs if isinstance(f, futures.GreenFuture)) if not green_fs: - return tuple(futures.wait(fs, timeout=timeout, - return_when=futures.FIRST_COMPLETED)) + return tuple(_futures.wait(fs, timeout=timeout, + return_when=_futures.FIRST_COMPLETED)) else: non_green_fs = len(fs) - green_fs if non_green_fs: @@ -40,11 +63,48 @@ def wait_for_any(fs, timeout=None): " non-green futures in the same `wait_for_any`" " call" % (green_fs, non_green_fs)) else: - return eu.wait_for_any(fs, timeout=timeout) + return _wait_for_any_green(fs, timeout=timeout) -def make_completed_future(result): - """Make with completed with given result.""" - future = futures.Future() - future.set_result(result) - return future +class _GreenWaiter(object): + """Provides the event that wait_for_any() blocks on.""" + def __init__(self): + self.event = greenthreading.Event() + + def add_result(self, future): + self.event.set() + + def add_exception(self, future): + self.event.set() + + def add_cancelled(self, future): + self.event.set() + + +def _wait_for_any_green(fs, timeout=None): + assert EVENTLET_AVAILABLE, 'eventlet is needed to wait on green futures' + + def _partition_futures(fs): + done = set() + not_done = set() + for f in fs: + if f._state in _DONE_STATES: + done.add(f) + else: + not_done.add(f) + return (done, not_done) + + with _base._AcquireFutures(fs): + (done, not_done) = _partition_futures(fs) + if done: + return (done, not_done) + waiter = _GreenWaiter() + for f in fs: + f._waiters.append(waiter) + + waiter.event.wait(timeout) + for f in fs: + f._waiters.remove(waiter) + + with _base._AcquireFutures(fs): + return _partition_futures(fs) From ca101d1abeddc4dc34c54c3b3e182bf008bb86c2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 14 Oct 2014 15:59:54 -0700 Subject: [PATCH 078/240] Use the mock that finds a working implementation Instead of using the library provided mock, use the import logic in our tests module that tries to find the best one that is usable in the current environment. It appears that this logic is *still* needed due to bugs in the non-bundled mock that causes errors/exceptions such as: AttributeError: 'method-wrapper' object has no attribute '__module__' It seems this is related to (or is this same) upstream bug: - https://code.google.com/p/mock/issues/detail?id=234 Change-Id: Ifeb3017f43b7d34af155ceca35d040706d34b185 --- taskflow/test.py | 7 +++++++ taskflow/tests/unit/test_utils_lock_utils.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/taskflow/test.py b/taskflow/test.py index 32c0b0fc..c6a56a99 100644 --- a/taskflow/test.py +++ b/taskflow/test.py @@ -21,13 +21,20 @@ import fixtures from oslotest import base from oslotest import mockpatch import six + +# This is weird like this since we want to import a mock that works the best +# and we need to try this import order, since oslotest registers a six.moves +# module (but depending on the import order of importing oslotest we may or +# may not see that change when trying to use it from six). try: from six.moves import mock except ImportError: try: + # In python 3.3+ mock got included in the standard library... from unittest import mock except ImportError: import mock + from testtools import compat from testtools import matchers from testtools import testcase diff --git a/taskflow/tests/unit/test_utils_lock_utils.py b/taskflow/tests/unit/test_utils_lock_utils.py index c7fe09f1..37bc1711 100644 --- a/taskflow/tests/unit/test_utils_lock_utils.py +++ b/taskflow/tests/unit/test_utils_lock_utils.py @@ -19,9 +19,9 @@ import threading import time from concurrent import futures -import mock from taskflow import test +from taskflow.test import mock from taskflow.tests import utils as test_utils from taskflow.utils import lock_utils from taskflow.utils import threading_utils From 7fe6bf0d6b3fa1dc0345ab0e9af4909c3f5b9549 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 21 Aug 2014 23:59:19 -0700 Subject: [PATCH 079/240] Remove attrdict and just use existing types In order to make it simpler (and less code) just prefer to use object types that already exist instead of trying to make dictionaries also behave like objects. For those that really need this kind of functionality: https://pypi.python.org/pypi/attrdict Change-Id: Ib7ddfa517f0500082fafac2c3e53fd6a158a6ddf --- taskflow/engines/helpers.py | 6 +- taskflow/engines/worker_based/proxy.py | 29 +++- taskflow/examples/fake_billing.py | 8 +- taskflow/jobs/backends/__init__.py | 6 +- taskflow/persistence/backends/__init__.py | 6 +- .../tests/unit/conductor/test_conductor.py | 11 +- taskflow/tests/unit/test_utils.py | 108 -------------- taskflow/utils/misc.py | 136 ++++-------------- 8 files changed, 74 insertions(+), 236 deletions(-) diff --git a/taskflow/engines/helpers.py b/taskflow/engines/helpers.py index a6236591..caf7ec13 100644 --- a/taskflow/engines/helpers.py +++ b/taskflow/engines/helpers.py @@ -54,12 +54,12 @@ def _extract_engine(**kwargs): kind = ENGINE_DEFAULT # See if it's a URI and if so, extract any further options... try: - pieces = misc.parse_uri(kind) + uri = misc.parse_uri(kind) except (TypeError, ValueError): pass else: - kind = pieces['scheme'] - options = misc.merge_uri(pieces, options.copy()) + kind = uri.scheme + options = misc.merge_uri(uri, options.copy()) # Merge in any leftover **kwargs into the options, this makes it so that # the provided **kwargs override any URI or engine_conf specific options. options.update(kwargs) diff --git a/taskflow/engines/worker_based/proxy.py b/taskflow/engines/worker_based/proxy.py index e0f9ce7d..6f608f42 100644 --- a/taskflow/engines/worker_based/proxy.py +++ b/taskflow/engines/worker_based/proxy.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +import collections import logging import socket @@ -21,7 +22,6 @@ import kombu import six from taskflow.engines.worker_based import dispatcher -from taskflow.utils import misc from taskflow.utils import threading_utils LOG = logging.getLogger(__name__) @@ -30,6 +30,16 @@ LOG = logging.getLogger(__name__) # the socket can get "stuck", and is a best practice for Kombu consumers. DRAIN_EVENTS_PERIOD = 1 +# Helper objects returned when requested to get connection details, used +# instead of returning the raw results from the kombu connection objects +# themselves so that a person can not mutate those objects (which would be +# bad). +_ConnectionDetails = collections.namedtuple('_ConnectionDetails', + ['uri', 'transport']) +_TransportDetails = collections.namedtuple('_TransportDetails', + ['options', 'driver_type', + 'driver_name', 'driver_version']) + class Proxy(object): """A proxy processes messages from/to the named exchange.""" @@ -71,13 +81,18 @@ class Proxy(object): driver_version = self._conn.transport.driver_version() if driver_version and driver_version.lower() == 'n/a': driver_version = None - return misc.AttrDict( + if self._conn.transport_options: + transport_options = self._conn.transport_options.copy() + else: + transport_options = {} + transport = _TransportDetails( + options=transport_options, + driver_type=self._conn.transport.driver_type, + driver_name=self._conn.transport.driver_name, + driver_version=driver_version) + return _ConnectionDetails( uri=self._conn.as_uri(include_password=False), - transport=misc.AttrDict( - options=dict(self._conn.transport_options), - driver_type=self._conn.transport.driver_type, - driver_name=self._conn.transport.driver_name, - driver_version=driver_version)) + transport=transport) @property def is_running(self): diff --git a/taskflow/examples/fake_billing.py b/taskflow/examples/fake_billing.py index ac15dbae..22c75cd9 100644 --- a/taskflow/examples/fake_billing.py +++ b/taskflow/examples/fake_billing.py @@ -148,6 +148,12 @@ class DeclareSuccess(task.Task): print("All data processed and sent to %s" % (sent_to)) +class DummyUser(object): + def __init__(self, user, id): + self.user = user + self.id = id + + # Resources (db handles and similar) of course can *not* be persisted so we # need to make sure that we pass this resource fetcher to the tasks constructor # so that the tasks have access to any needed resources (the resources are @@ -168,7 +174,7 @@ flow.add(sub_flow) # prepopulating this allows the tasks that dependent on the 'request' variable # to start processing (in this case this is the ExtractInputRequest task). store = { - 'request': misc.AttrDict(user="bob", id="1.35"), + 'request': DummyUser(user="bob", id="1.35"), } eng = engines.load(flow, engine='serial', store=store) diff --git a/taskflow/jobs/backends/__init__.py b/taskflow/jobs/backends/__init__.py index 099f0476..94afa6e5 100644 --- a/taskflow/jobs/backends/__init__.py +++ b/taskflow/jobs/backends/__init__.py @@ -55,12 +55,12 @@ def fetch(name, conf, namespace=BACKEND_NAMESPACE, **kwargs): conf = {'board': conf} board = conf['board'] try: - pieces = misc.parse_uri(board) + uri = misc.parse_uri(board) except (TypeError, ValueError): pass else: - board = pieces['scheme'] - conf = misc.merge_uri(pieces, conf.copy()) + board = uri.scheme + conf = misc.merge_uri(uri, conf.copy()) LOG.debug('Looking for %r jobboard driver in %r', board, namespace) try: mgr = driver.DriverManager(namespace, board, diff --git a/taskflow/persistence/backends/__init__.py b/taskflow/persistence/backends/__init__.py index 6faabdef..64b7cda1 100644 --- a/taskflow/persistence/backends/__init__.py +++ b/taskflow/persistence/backends/__init__.py @@ -52,12 +52,12 @@ def fetch(conf, namespace=BACKEND_NAMESPACE, **kwargs): """ backend_name = conf['connection'] try: - pieces = misc.parse_uri(backend_name) + uri = misc.parse_uri(backend_name) except (TypeError, ValueError): pass else: - backend_name = pieces['scheme'] - conf = misc.merge_uri(pieces, conf.copy()) + backend_name = uri.scheme + conf = misc.merge_uri(uri, conf.copy()) LOG.debug('Looking for %r backend driver in %r', backend_name, namespace) try: mgr = driver.DriverManager(namespace, backend_name, diff --git a/taskflow/tests/unit/conductor/test_conductor.py b/taskflow/tests/unit/conductor/test_conductor.py index 8f9c2d10..cf19fa84 100644 --- a/taskflow/tests/unit/conductor/test_conductor.py +++ b/taskflow/tests/unit/conductor/test_conductor.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +import collections import contextlib import threading @@ -28,7 +29,6 @@ from taskflow.persistence.backends import impl_memory from taskflow import states as st from taskflow import test from taskflow.tests import utils as test_utils -from taskflow.utils import misc from taskflow.utils import persistence_utils as pu from taskflow.utils import threading_utils @@ -58,6 +58,10 @@ def make_thread(conductor): class SingleThreadedConductorTest(test_utils.EngineTestBase, test.TestCase): + ComponentBundle = collections.namedtuple('ComponentBundle', + ['board', 'client', + 'persistence', 'conductor']) + def make_components(self, name='testing', wait_timeout=0.1): client = fake_client.FakeClient() persistence = impl_memory.MemoryBackend() @@ -66,10 +70,7 @@ class SingleThreadedConductorTest(test_utils.EngineTestBase, test.TestCase): persistence=persistence) conductor = stc.SingleThreadedConductor(name, board, persistence, wait_timeout=wait_timeout) - return misc.AttrDict(board=board, - client=client, - persistence=persistence, - conductor=conductor) + return self.ComponentBundle(board, client, persistence, conductor) def test_connection(self): components = self.make_components() diff --git a/taskflow/tests/unit/test_utils.py b/taskflow/tests/unit/test_utils.py index e8660bd3..38417810 100644 --- a/taskflow/tests/unit/test_utils.py +++ b/taskflow/tests/unit/test_utils.py @@ -387,109 +387,6 @@ class CachedPropertyTest(test.TestCase): self.assertEqual('b', a.b) -class AttrDictTest(test.TestCase): - def test_ok_create(self): - attrs = { - 'a': 1, - 'b': 2, - } - obj = misc.AttrDict(**attrs) - self.assertEqual(obj.a, 1) - self.assertEqual(obj.b, 2) - - def test_private_create(self): - attrs = { - '_a': 1, - } - self.assertRaises(AttributeError, misc.AttrDict, **attrs) - - def test_invalid_create(self): - attrs = { - # Python attributes can't start with a number. - '123_abc': 1, - } - self.assertRaises(AttributeError, misc.AttrDict, **attrs) - - def test_no_overwrite(self): - attrs = { - # Python attributes can't start with a number. - 'update': 1, - } - self.assertRaises(AttributeError, misc.AttrDict, **attrs) - - def test_back_todict(self): - attrs = { - 'a': 1, - } - obj = misc.AttrDict(**attrs) - self.assertEqual(obj.a, 1) - self.assertEqual(attrs, dict(obj)) - - def test_runtime_invalid_set(self): - - def bad_assign(obj): - obj._123 = 'b' - - attrs = { - 'a': 1, - } - obj = misc.AttrDict(**attrs) - self.assertEqual(obj.a, 1) - self.assertRaises(AttributeError, bad_assign, obj) - - def test_bypass_get(self): - attrs = { - 'a': 1, - } - obj = misc.AttrDict(**attrs) - self.assertEqual(1, obj['a']) - - def test_bypass_set_no_get(self): - - def bad_assign(obj): - obj._b = 'e' - - attrs = { - 'a': 1, - } - obj = misc.AttrDict(**attrs) - self.assertEqual(1, obj['a']) - obj['_b'] = 'c' - self.assertRaises(AttributeError, bad_assign, obj) - self.assertEqual('c', obj['_b']) - - -class IsValidAttributeNameTestCase(test.TestCase): - def test_a_is_ok(self): - self.assertTrue(misc.is_valid_attribute_name('a')) - - def test_name_can_be_longer(self): - self.assertTrue(misc.is_valid_attribute_name('foobarbaz')) - - def test_name_can_have_digits(self): - self.assertTrue(misc.is_valid_attribute_name('fo12')) - - def test_name_cannot_start_with_digit(self): - self.assertFalse(misc.is_valid_attribute_name('1z')) - - def test_hidden_names_are_forbidden(self): - self.assertFalse(misc.is_valid_attribute_name('_z')) - - def test_hidden_names_can_be_allowed(self): - self.assertTrue( - misc.is_valid_attribute_name('_z', allow_hidden=True)) - - def test_self_is_forbidden(self): - self.assertFalse(misc.is_valid_attribute_name('self')) - - def test_self_can_be_allowed(self): - self.assertTrue( - misc.is_valid_attribute_name('self', allow_self=True)) - - def test_no_unicode_please(self): - self.assertFalse(misc.is_valid_attribute_name('mañana')) - - class UriParseTest(test.TestCase): def test_parse(self): url = "zookeeper://192.168.0.1:2181/a/b/?c=d" @@ -501,11 +398,6 @@ class UriParseTest(test.TestCase): self.assertEqual('/a/b/', parsed.path) self.assertEqual({'c': 'd'}, parsed.params) - def test_multi_params(self): - url = "mysql://www.yahoo.com:3306/a/b/?c=d&c=e" - parsed = misc.parse_uri(url, query_duplicates=True) - self.assertEqual({'c': ['d', 'e']}, parsed.params) - def test_port_provided(self): url = "rabbitmq://www.yahoo.com:5672" parsed = misc.parse_uri(url) diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index a4619881..3822188c 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -19,11 +19,9 @@ import contextlib import datetime import errno import inspect -import keyword import logging import os import re -import string import sys import threading @@ -48,7 +46,23 @@ NUMERIC_TYPES = six.integer_types + (float,) _SCHEME_REGEX = re.compile(r"^([A-Za-z][A-Za-z0-9+.-]*):") -def merge_uri(uri_pieces, conf): +# FIXME(harlowja): This should be removed with the next version of oslo.utils +# which now has this functionality built-in, until then we are deriving from +# there base class and adding this functionality on... +# +# The change was merged @ https://review.openstack.org/#/c/118881/ +class ModifiedSplitResult(netutils._ModifiedSplitResult): + """A split result that exposes the query parameters as a dictionary.""" + + @property + def params(self): + if self.query: + return dict(urlparse.parse_qsl(self.query)) + else: + return {} + + +def merge_uri(uri, conf): """Merges a parsed uri into the given configuration dictionary. Merges the username, password, hostname, and query params of a uri into @@ -57,22 +71,21 @@ def merge_uri(uri_pieces, conf): NOTE(harlowja): does not merge the path, scheme or fragment. """ - for k in ('username', 'password'): - if not uri_pieces[k]: + for (k, v) in [('username', uri.username), ('password', uri.password)]: + if not v: continue - conf.setdefault(k, uri_pieces[k]) - hostname = uri_pieces.get('hostname') - if hostname: - port = uri_pieces.get('port') - if port is not None: - hostname += ":%s" % (port) + conf.setdefault(k, v) + if uri.hostname: + hostname = uri.hostname + if uri.port is not None: + hostname += ":%s" % (uri.port) conf.setdefault('hostname', hostname) - for (k, v) in six.iteritems(uri_pieces['params']): + for (k, v) in six.iteritems(uri.params): conf.setdefault(k, v) return conf -def parse_uri(uri, query_duplicates=False): +def parse_uri(uri): """Parses a uri into its components.""" # Do some basic validation before continuing... if not isinstance(uri, six.string_types): @@ -83,38 +96,10 @@ def parse_uri(uri, query_duplicates=False): if not match: raise ValueError("Uri %r does not start with a RFC 3986 compliant" " scheme" % (uri)) - parsed = netutils.urlsplit(uri) - if parsed.query: - query_params = urlparse.parse_qsl(parsed.query) - if not query_duplicates: - query_params = dict(query_params) - else: - # Retain duplicates in a list for keys which have duplicates, but - # for items which are not duplicated, just associate the key with - # the value. - tmp_query_params = {} - for (k, v) in query_params: - if k in tmp_query_params: - p_v = tmp_query_params[k] - if isinstance(p_v, list): - p_v.append(v) - else: - p_v = [p_v, v] - tmp_query_params[k] = p_v - else: - tmp_query_params[k] = v - query_params = tmp_query_params - else: - query_params = {} - return AttrDict( - scheme=parsed.scheme, - username=parsed.username, - password=parsed.password, - fragment=parsed.fragment, - path=parsed.path, - params=query_params, - hostname=parsed.hostname, - port=parsed.port) + split = netutils.urlsplit(uri) + return ModifiedSplitResult(scheme=split.scheme, fragment=split.fragment, + path=split.path, netloc=split.netloc, + query=split.query) def binary_encode(text, encoding='utf-8'): @@ -271,67 +256,6 @@ def get_duplicate_keys(iterable, key=None): return duplicates -# NOTE(imelnikov): we should not use str.isalpha or str.isdigit -# as they are locale-dependant -_ASCII_WORD_SYMBOLS = frozenset(string.ascii_letters + string.digits + '_') - - -def is_valid_attribute_name(name, allow_self=False, allow_hidden=False): - """Checks that a string is a valid/invalid python attribute name.""" - return all(( - isinstance(name, six.string_types), - len(name) > 0, - (allow_self or not name.lower().startswith('self')), - (allow_hidden or not name.lower().startswith('_')), - - # NOTE(imelnikov): keywords should be forbidden. - not keyword.iskeyword(name), - - # See: http://docs.python.org/release/2.5.2/ref/grammar.txt - not (name[0] in string.digits), - all(symbol in _ASCII_WORD_SYMBOLS for symbol in name) - )) - - -class AttrDict(dict): - """Dictionary subclass that allows for attribute based access. - - This subclass allows for accessing a dictionaries keys and values by - accessing those keys as regular attributes. Keys that are not valid python - attribute names can not of course be acccessed/set (those keys must be - accessed/set by the traditional dictionary indexing operators instead). - """ - NO_ATTRS = tuple(reflection.get_member_names(dict)) - - @classmethod - def _is_valid_attribute_name(cls, name): - if not is_valid_attribute_name(name): - return False - # Make the name just be a simple string in latin-1 encoding in python3. - if name in cls.NO_ATTRS: - return False - return True - - def __init__(self, **kwargs): - for (k, v) in kwargs.items(): - if not self._is_valid_attribute_name(k): - raise AttributeError("Invalid attribute name: '%s'" % (k)) - self[k] = v - - def __getattr__(self, name): - if not self._is_valid_attribute_name(name): - raise AttributeError("Invalid attribute name: '%s'" % (name)) - try: - return self[name] - except KeyError: - raise AttributeError("No attributed named: '%s'" % (name)) - - def __setattr__(self, name, value): - if not self._is_valid_attribute_name(name): - raise AttributeError("Invalid attribute name: '%s'" % (name)) - self[name] = value - - class ExponentialBackoff(object): """An iterable object that will yield back an exponential delay sequence. From 58f27fcd2d7d3206477cbf58e851ba208ba38975 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 18 Oct 2014 18:38:28 -0700 Subject: [PATCH 080/240] Remove direct usage of the deprecated notifier location Internally we should be using the new location and not the deprecated location wherever possible. This avoids emitting warnings messages on our own code, which is a dirty habit. Change-Id: Icc389e61613bc78e64083f0086f6f23aabd243d1 --- taskflow/engines/base.py | 5 +++-- taskflow/examples/delayed_return.py | 4 ++-- taskflow/jobs/jobboard.py | 4 ++-- taskflow/listeners/base.py | 5 +++-- taskflow/listeners/logging.py | 9 +++++---- taskflow/listeners/printing.py | 6 +++--- taskflow/tests/unit/action_engine/test_runner.py | 4 ++-- taskflow/utils/misc.py | 2 +- 8 files changed, 21 insertions(+), 18 deletions(-) diff --git a/taskflow/engines/base.py b/taskflow/engines/base.py index 63389fe0..2fab9d0f 100644 --- a/taskflow/engines/base.py +++ b/taskflow/engines/base.py @@ -19,6 +19,7 @@ import abc import six +from taskflow.types import notifier from taskflow.utils import misc @@ -40,8 +41,8 @@ class EngineBase(object): self._options = {} else: self._options = dict(options) - self.notifier = misc.Notifier() - self.task_notifier = misc.Notifier() + self.notifier = notifier.Notifier() + self.task_notifier = notifier.Notifier() @property def options(self): diff --git a/taskflow/examples/delayed_return.py b/taskflow/examples/delayed_return.py index bc44a897..5ca70078 100644 --- a/taskflow/examples/delayed_return.py +++ b/taskflow/examples/delayed_return.py @@ -39,14 +39,14 @@ from taskflow.listeners import base from taskflow.patterns import linear_flow as lf from taskflow import states from taskflow import task -from taskflow.utils import misc +from taskflow.types import notifier class PokeFutureListener(base.ListenerBase): def __init__(self, engine, future, task_name): super(PokeFutureListener, self).__init__( engine, - task_listen_for=(misc.Notifier.ANY,), + task_listen_for=(notifier.Notifier.ANY,), flow_listen_for=[]) self._future = future self._task_name = task_name diff --git a/taskflow/jobs/jobboard.py b/taskflow/jobs/jobboard.py index d7d0850f..0938d0e7 100644 --- a/taskflow/jobs/jobboard.py +++ b/taskflow/jobs/jobboard.py @@ -19,7 +19,7 @@ import abc import six -from taskflow.utils import misc +from taskflow.types import notifier @six.add_metaclass(abc.ABCMeta) @@ -203,4 +203,4 @@ class NotifyingJobBoard(JobBoard): """ def __init__(self, name, conf): super(NotifyingJobBoard, self).__init__(name, conf) - self.notifier = misc.Notifier() + self.notifier = notifier.Notifier() diff --git a/taskflow/listeners/base.py b/taskflow/listeners/base.py index f11202f5..ca2ee6e2 100644 --- a/taskflow/listeners/base.py +++ b/taskflow/listeners/base.py @@ -23,6 +23,7 @@ from oslo.utils import excutils import six from taskflow import states +from taskflow.types import notifier from taskflow.utils import misc LOG = logging.getLogger(__name__) @@ -46,8 +47,8 @@ class ListenerBase(object): """ def __init__(self, engine, - task_listen_for=(misc.Notifier.ANY,), - flow_listen_for=(misc.Notifier.ANY,)): + task_listen_for=(notifier.Notifier.ANY,), + flow_listen_for=(notifier.Notifier.ANY,)): if not task_listen_for: task_listen_for = [] if not flow_listen_for: diff --git a/taskflow/listeners/logging.py b/taskflow/listeners/logging.py index 175bc6fa..87528f37 100644 --- a/taskflow/listeners/logging.py +++ b/taskflow/listeners/logging.py @@ -21,6 +21,7 @@ import sys from taskflow.listeners import base from taskflow import states +from taskflow.types import notifier from taskflow.utils import misc LOG = logging.getLogger(__name__) @@ -51,8 +52,8 @@ class LoggingListener(base.LoggingBase): is provided. """ def __init__(self, engine, - task_listen_for=(misc.Notifier.ANY,), - flow_listen_for=(misc.Notifier.ANY,), + task_listen_for=(notifier.Notifier.ANY,), + flow_listen_for=(notifier.Notifier.ANY,), log=None, level=logging.DEBUG): super(LoggingListener, self).__init__(engine, @@ -99,8 +100,8 @@ class DynamicLoggingListener(base.ListenerBase): """ def __init__(self, engine, - task_listen_for=(misc.Notifier.ANY,), - flow_listen_for=(misc.Notifier.ANY,), + task_listen_for=(notifier.Notifier.ANY,), + flow_listen_for=(notifier.Notifier.ANY,), log=None, failure_level=logging.WARNING, level=logging.DEBUG): super(DynamicLoggingListener, self).__init__( diff --git a/taskflow/listeners/printing.py b/taskflow/listeners/printing.py index e9359bf5..a7a137b1 100644 --- a/taskflow/listeners/printing.py +++ b/taskflow/listeners/printing.py @@ -20,14 +20,14 @@ import sys import traceback from taskflow.listeners import base -from taskflow.utils import misc +from taskflow.types import notifier class PrintingListener(base.LoggingBase): """Writes the task and flow notifications messages to stdout or stderr.""" def __init__(self, engine, - task_listen_for=(misc.Notifier.ANY,), - flow_listen_for=(misc.Notifier.ANY,), + task_listen_for=(notifier.Notifier.ANY,), + flow_listen_for=(notifier.Notifier.ANY,), stderr=False): super(PrintingListener, self).__init__(engine, task_listen_for=task_listen_for, diff --git a/taskflow/tests/unit/action_engine/test_runner.py b/taskflow/tests/unit/action_engine/test_runner.py index eca1e6cc..9b3bdb47 100644 --- a/taskflow/tests/unit/action_engine/test_runner.py +++ b/taskflow/tests/unit/action_engine/test_runner.py @@ -27,7 +27,7 @@ from taskflow import storage from taskflow import test from taskflow.tests import utils as test_utils from taskflow.types import fsm -from taskflow.utils import misc +from taskflow.types import notifier from taskflow.utils import persistence_utils as pu @@ -41,7 +41,7 @@ class _RunnerTestMixin(object): store.ensure_atom(task) if initial_state: store.set_flow_state(initial_state) - task_notifier = misc.Notifier() + task_notifier = notifier.Notifier() task_executor = executor.SerialTaskExecutor() task_executor.start() self.addCleanup(task_executor.stop) diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index a4619881..a71c94be 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -428,4 +428,4 @@ def capture_failure(): if not any(exc_info): raise RuntimeError("No active exception is being handled") else: - yield Failure(exc_info=exc_info) + yield failure.Failure(exc_info=exc_info) From ac8eefd0e61672884077c31ceab025d99796f80d Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 19 Oct 2014 20:37:16 -0700 Subject: [PATCH 081/240] Use constants for retry automatically provided kwargs Instead of using strings use module level constants for the automatically provided keyword arguments to the retry revert/execution functions. This makes it easier for users of taskflow to associate these constants with the actual keywords, without having to resort to using raw strings directly. Change-Id: I739a0ad69819c3ca0d10c98966012cf9ef1b86bd --- taskflow/engines/action_engine/retry_action.py | 18 ++++++++++++------ taskflow/retry.py | 11 ++++++++++- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/taskflow/engines/action_engine/retry_action.py b/taskflow/engines/action_engine/retry_action.py index 3bf6f491..7b55ffab 100644 --- a/taskflow/engines/action_engine/retry_action.py +++ b/taskflow/engines/action_engine/retry_action.py @@ -17,6 +17,7 @@ import logging from taskflow.engines.action_engine import executor as ex +from taskflow import retry as rt from taskflow import states from taskflow.types import futures from taskflow.utils import misc @@ -33,12 +34,15 @@ class RetryAction(object): self._walker_factory = walker_factory self._executor = futures.SynchronousExecutor() - def _get_retry_args(self, retry): + def _get_retry_args(self, retry, addons=None): scope_walker = self._walker_factory(retry) kwargs = self._storage.fetch_mapped_args(retry.rebind, atom_name=retry.name, scope_walker=scope_walker) - kwargs['history'] = self._storage.get_retry_history(retry.name) + history = self._storage.get_retry_history(retry.name) + kwargs[rt.EXECUTE_REVERT_HISTORY] = history + if addons: + kwargs.update(addons) return kwargs def change_state(self, retry, state, result=None): @@ -83,8 +87,7 @@ class RetryAction(object): def revert(self, retry): - def _execute_retry(kwargs, failures): - kwargs['flow_failures'] = failures + def _execute_retry(kwargs): try: result = retry.revert(**kwargs) except Exception: @@ -99,9 +102,12 @@ class RetryAction(object): self.change_state(retry, states.REVERTED) self.change_state(retry, states.REVERTING) + arg_addons = { + rt.REVERT_FLOW_FAILURES: self._storage.get_failures(), + } fut = self._executor.submit(_execute_retry, - self._get_retry_args(retry), - self._storage.get_failures()) + self._get_retry_args(retry, + addons=arg_addons)) fut.add_done_callback(_on_done_callback) return fut diff --git a/taskflow/retry.py b/taskflow/retry.py index edfc4d18..6897e3b7 100644 --- a/taskflow/retry.py +++ b/taskflow/retry.py @@ -31,6 +31,15 @@ REVERT = "REVERT" REVERT_ALL = "REVERT_ALL" RETRY = "RETRY" +# Constants passed into revert/execute kwargs. +# +# Contains information about the past decisions and outcomes that have +# occurred (if available). +EXECUTE_REVERT_HISTORY = 'history' +# +# The cause of the flow failure/s +REVERT_FLOW_FAILURES = 'flow_failures' + @six.add_metaclass(abc.ABCMeta) class Retry(atom.Atom): @@ -56,7 +65,7 @@ class Retry(atom.Atom): provides = self.default_provides super(Retry, self).__init__(name, provides) self._build_arg_mapping(self.execute, requires, rebind, auto_extract, - ignore_list=['history']) + ignore_list=[EXECUTE_REVERT_HISTORY]) @property def name(self): From 1f12ab306fed42625b39a155476bd1da0865d480 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 20 Oct 2014 11:40:05 -0700 Subject: [PATCH 082/240] Fix the example 'default_provides' The example should be declaring that the whole result is 'a' and not just the first element of the returned result (which in this case is the string index[0] which is also 'a') to avoid propagating bad/broken patterns in the examples. Fixes bug 1383422 Change-Id: I3a39bb25291276d29ee260e84aab9ec182499fd9 --- taskflow/examples/simple_linear_pass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskflow/examples/simple_linear_pass.py b/taskflow/examples/simple_linear_pass.py index bda25216..d378418d 100644 --- a/taskflow/examples/simple_linear_pass.py +++ b/taskflow/examples/simple_linear_pass.py @@ -36,7 +36,7 @@ from taskflow import task class TaskA(task.Task): - default_provides = ['a'] + default_provides = 'a' def execute(self): print("Executing '%s'" % (self.name)) From 3c9871d8c3e5746db23a8749df0123818c6d9f55 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 18 Oct 2014 19:22:08 -0700 Subject: [PATCH 083/240] Remove direct usage of the deprecated failure location Internally we should be using the new location and not the deprecated location wherever possible. This avoids emitting warnings messages on our own code, which is a dirty habit. Change-Id: Idac5a772eca7529d92542ada3be1cea092880e25 --- doc/source/arguments_and_results.rst | 16 ++++---- doc/source/workers.rst | 16 ++++---- taskflow/engines/action_engine/engine.py | 5 ++- taskflow/engines/action_engine/executor.py | 6 +-- .../engines/action_engine/retry_action.py | 10 ++--- taskflow/engines/action_engine/runner.py | 8 ++-- taskflow/engines/action_engine/runtime.py | 5 ++- taskflow/engines/action_engine/task_action.py | 6 +-- taskflow/engines/worker_based/executor.py | 6 +-- taskflow/engines/worker_based/protocol.py | 10 ++--- taskflow/engines/worker_based/server.py | 13 +++--- taskflow/examples/wrapped_exception.py | 18 ++++----- taskflow/exceptions.py | 4 +- taskflow/listeners/base.py | 4 +- taskflow/listeners/logging.py | 8 ++-- .../persistence/backends/impl_sqlalchemy.py | 3 +- taskflow/persistence/logbook.py | 10 ++--- taskflow/storage.py | 3 +- taskflow/tests/unit/persistence/base.py | 15 ++++--- taskflow/tests/unit/test_engines.py | 4 +- taskflow/tests/unit/test_failure.py | 3 +- taskflow/tests/unit/test_storage.py | 40 +++++++++---------- .../tests/unit/worker_based/test_executor.py | 8 ++-- .../tests/unit/worker_based/test_pipeline.py | 4 +- .../tests/unit/worker_based/test_protocol.py | 15 +++---- .../tests/unit/worker_based/test_server.py | 26 ++++++------ taskflow/tests/utils.py | 4 +- taskflow/types/failure.py | 2 +- 28 files changed, 138 insertions(+), 134 deletions(-) diff --git a/doc/source/arguments_and_results.rst b/doc/source/arguments_and_results.rst index d7b96095..3082e2fc 100644 --- a/doc/source/arguments_and_results.rst +++ b/doc/source/arguments_and_results.rst @@ -350,7 +350,7 @@ For ``result`` value, two cases are possible: * If the task is being reverted because it failed (an exception was raised from its |task.execute| method), the ``result`` value is an instance of a - :py:class:`~taskflow.utils.misc.Failure` object that holds the exception + :py:class:`~taskflow.types.failure.Failure` object that holds the exception information. * If the task is being reverted because some other task failed, and this task @@ -361,9 +361,9 @@ All other arguments are fetched from storage in the same way it is done for |task.execute| method. To determine if a task failed you can check whether ``result`` is instance of -:py:class:`~taskflow.utils.misc.Failure`:: +:py:class:`~taskflow.types.failure.Failure`:: - from taskflow.utils import misc + from taskflow.types import failure class RevertingTask(task.Task): @@ -371,7 +371,7 @@ To determine if a task failed you can check whether ``result`` is instance of return do_something(spam, eggs) def revert(self, result, spam, eggs): - if isinstance(result, misc.Failure): + if isinstance(result, failure.Failure): print("This task failed, exception: %s" % result.exception_str) else: @@ -389,7 +389,7 @@ A |Retry| controller works with arguments in the same way as a |Task|. But it has an additional parameter ``'history'`` that is a list of tuples. Each tuple contains a result of the previous retry run and a table where the key is a failed task and the value is a -:py:class:`~taskflow.utils.misc.Failure` object. +:py:class:`~taskflow.types.failure.Failure` object. Consider the following implementation:: @@ -412,7 +412,7 @@ Imagine the above retry had returned a value ``'5'`` and then some task ``'A'`` failed with some exception. In this case the above retrys ``on_failure`` method will receive the following history:: - [('5', {'A': misc.Failure()})] + [('5', {'A': failure.Failure()})] At this point (since the implementation returned ``RETRY``) the |retry.execute| method will be called again and it will receive the same @@ -421,10 +421,10 @@ there behavior. If instead the |retry.execute| method raises an exception, the |retry.revert| method of the implementation will be called and -a :py:class:`~taskflow.utils.misc.Failure` object will be present in the +a :py:class:`~taskflow.types.failure.Failure` object will be present in the history instead of the typical result:: - [('5', {'A': misc.Failure()}), (misc.Failure(), {})] + [('5', {'A': failure.Failure()}), (failure.Failure(), {})] .. note:: diff --git a/doc/source/workers.rst b/doc/source/workers.rst index caac6aa0..4bb5b362 100644 --- a/doc/source/workers.rst +++ b/doc/source/workers.rst @@ -135,7 +135,7 @@ engine executor in the following manner: executes the task). 2. If dispatched succeeded then the worker sends a confirmation response to the executor otherwise the worker sends a failed response along with - a serialized :py:class:`failure ` object + a serialized :py:class:`failure ` object that contains what has failed (and why). 3. The worker executes the task and once it is finished sends the result back to the originating executor (every time a task progress event is @@ -152,11 +152,11 @@ engine executor in the following manner: .. note:: - :py:class:`~taskflow.utils.misc.Failure` objects are not json-serializable - (they contain references to tracebacks which are not serializable), so they - are converted to dicts before sending and converted from dicts after - receiving on both executor & worker sides (this translation is lossy since - the traceback won't be fully retained). + :py:class:`~taskflow.types.failure.Failure` objects are not directly + json-serializable (they contain references to tracebacks which are not + serializable), so they are converted to dicts before sending and converted + from dicts after receiving on both executor & worker sides (this + translation is lossy since the traceback won't be fully retained). Executor request format ~~~~~~~~~~~~~~~~~~~~~~~ @@ -165,7 +165,7 @@ Executor request format * **action** - task action to be performed (e.g. execute, revert) * **arguments** - arguments the task action to be called with * **result** - task execution result (result or - :py:class:`~taskflow.utils.misc.Failure`) *[passed to revert only]* + :py:class:`~taskflow.types.failure.Failure`) *[passed to revert only]* Additionally, the following parameters are added to the request message: @@ -222,7 +222,7 @@ When **failed:** { "event": , - "result": , + "result": , "state": "FAILURE" } diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index 22db3ed7..a501c45d 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -26,6 +26,7 @@ from taskflow.engines import base from taskflow import exceptions as exc from taskflow import states from taskflow import storage as atom_storage +from taskflow.types import failure from taskflow.utils import lock_utils from taskflow.utils import misc from taskflow.utils import reflection @@ -129,7 +130,7 @@ class ActionEngine(base.EngineBase): closed = False for (last_state, failures) in runner.run_iter(timeout=timeout): if failures: - misc.Failure.reraise_if_any(failures) + failure.Failure.reraise_if_any(failures) if closed: continue try: @@ -152,7 +153,7 @@ class ActionEngine(base.EngineBase): self._change_state(last_state) if last_state not in [states.SUSPENDED, states.SUCCESS]: failures = self.storage.get_failures() - misc.Failure.reraise_if_any(failures.values()) + failure.Failure.reraise_if_any(failures.values()) def _change_state(self, state): with self._state_lock: diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index 83da3b6b..78d16ff9 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -19,9 +19,9 @@ import abc import six from taskflow import task as _task +from taskflow.types import failure from taskflow.types import futures from taskflow.utils import async_utils -from taskflow.utils import misc from taskflow.utils import threading_utils # Execution and reversion events. @@ -37,7 +37,7 @@ def _execute_task(task, arguments, progress_callback): except Exception: # NOTE(imelnikov): wrap current exception with Failure # object and return it. - result = misc.Failure() + result = failure.Failure() finally: task.post_execute() return (task, EXECUTED, result) @@ -54,7 +54,7 @@ def _revert_task(task, arguments, result, failures, progress_callback): except Exception: # NOTE(imelnikov): wrap current exception with Failure # object and return it. - result = misc.Failure() + result = failure.Failure() finally: task.post_revert() return (task, REVERTED, result) diff --git a/taskflow/engines/action_engine/retry_action.py b/taskflow/engines/action_engine/retry_action.py index 3bf6f491..0ffa4b2a 100644 --- a/taskflow/engines/action_engine/retry_action.py +++ b/taskflow/engines/action_engine/retry_action.py @@ -18,8 +18,8 @@ import logging from taskflow.engines.action_engine import executor as ex from taskflow import states +from taskflow.types import failure from taskflow.types import futures -from taskflow.utils import misc LOG = logging.getLogger(__name__) @@ -65,12 +65,12 @@ class RetryAction(object): try: result = retry.execute(**kwargs) except Exception: - result = misc.Failure() + result = failure.Failure() return (retry, ex.EXECUTED, result) def _on_done_callback(fut): result = fut.result()[-1] - if isinstance(result, misc.Failure): + if isinstance(result, failure.Failure): self.change_state(retry, states.FAILURE, result=result) else: self.change_state(retry, states.SUCCESS, result=result) @@ -88,12 +88,12 @@ class RetryAction(object): try: result = retry.revert(**kwargs) except Exception: - result = misc.Failure() + result = failure.Failure() return (retry, ex.REVERTED, result) def _on_done_callback(fut): result = fut.result()[-1] - if isinstance(result, misc.Failure): + if isinstance(result, failure.Failure): self.change_state(retry, states.FAILURE) else: self.change_state(retry, states.REVERTED) diff --git a/taskflow/engines/action_engine/runner.py b/taskflow/engines/action_engine/runner.py index c2b1788a..79ebc657 100644 --- a/taskflow/engines/action_engine/runner.py +++ b/taskflow/engines/action_engine/runner.py @@ -17,8 +17,8 @@ import logging from taskflow import states as st +from taskflow.types import failure from taskflow.types import fsm -from taskflow.utils import misc # Waiting state timeout (in seconds). _WAITING_TIMEOUT = 60 @@ -132,15 +132,15 @@ class _MachineBuilder(object): try: node, event, result = fut.result() retain = self._completer.complete(node, event, result) - if retain and isinstance(result, misc.Failure): + if retain and isinstance(result, failure.Failure): memory.failures.append(result) except Exception: - memory.failures.append(misc.Failure()) + memory.failures.append(failure.Failure()) else: try: more_nodes = self._analyzer.get_next_nodes(node) except Exception: - memory.failures.append(misc.Failure()) + memory.failures.append(failure.Failure()) else: next_nodes.update(more_nodes) if self.runnable() and next_nodes and not memory.failures: diff --git a/taskflow/engines/action_engine/runtime.py b/taskflow/engines/action_engine/runtime.py index c0c58367..06959f29 100644 --- a/taskflow/engines/action_engine/runtime.py +++ b/taskflow/engines/action_engine/runtime.py @@ -24,6 +24,7 @@ from taskflow import exceptions as excp from taskflow import retry as retry_atom from taskflow import states as st from taskflow import task as task_atom +from taskflow.types import failure from taskflow.utils import misc @@ -155,7 +156,7 @@ class Completer(object): """ if isinstance(node, task_atom.BaseTask): self._complete_task(node, event, result) - if isinstance(result, misc.Failure): + if isinstance(result, failure.Failure): if event == ex.EXECUTED: self._process_atom_failure(node, result) else: @@ -270,5 +271,5 @@ class Scheduler(object): # Immediately stop scheduling future work so that we can # exit execution early (rather than later) if a single task # fails to schedule correctly. - return (futures, [misc.Failure()]) + return (futures, [failure.Failure()]) return (futures, []) diff --git a/taskflow/engines/action_engine/task_action.py b/taskflow/engines/action_engine/task_action.py index 3503df7c..eb9510f9 100644 --- a/taskflow/engines/action_engine/task_action.py +++ b/taskflow/engines/action_engine/task_action.py @@ -17,7 +17,7 @@ import logging from taskflow import states -from taskflow.utils import misc +from taskflow.types import failure LOG = logging.getLogger(__name__) @@ -91,7 +91,7 @@ class TaskAction(object): self._on_update_progress) def complete_execution(self, task, result): - if isinstance(result, misc.Failure): + if isinstance(result, failure.Failure): self.change_state(task, states.FAILURE, result=result) else: self.change_state(task, states.SUCCESS, @@ -112,7 +112,7 @@ class TaskAction(object): return future def complete_reversion(self, task, rev_result): - if isinstance(rev_result, misc.Failure): + if isinstance(rev_result, failure.Failure): self.change_state(task, states.FAILURE) else: self.change_state(task, states.REVERTED, progress=1.0) diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index 235f3c93..ae8e0e40 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -172,9 +172,9 @@ class WorkerTaskExecutor(executor.TaskExecutorBase): " seconds for it to transition out of (%s) states" % (request, request_age, ", ".join(pr.WAITING_STATES))) except exc.RequestTimeout: - with misc.capture_failure() as fail: - LOG.debug(fail.exception_str) - request.set_result(fail) + with misc.capture_failure() as failure: + LOG.debug(failure.exception_str) + request.set_result(failure) def _on_wait(self): """This function is called cyclically between draining events.""" diff --git a/taskflow/engines/worker_based/protocol.py b/taskflow/engines/worker_based/protocol.py index a97240a9..3cd7e178 100644 --- a/taskflow/engines/worker_based/protocol.py +++ b/taskflow/engines/worker_based/protocol.py @@ -26,9 +26,9 @@ import six from taskflow.engines.action_engine import executor from taskflow import exceptions as excp +from taskflow.types import failure as ft from taskflow.types import timing as tt from taskflow.utils import lock_utils -from taskflow.utils import misc from taskflow.utils import reflection # NOTE(skudriashev): This is protocol states and events, which are not @@ -270,15 +270,15 @@ class Request(Message): """Return json-serializable request. To convert requests that have failed due to some exception this will - convert all `misc.Failure` objects into dictionaries (which will then - be reconstituted by the receiver). + convert all `failure.Failure` objects into dictionaries (which will + then be reconstituted by the receiver). """ request = dict(task_cls=self._task_cls, task_name=self._task.name, task_version=self._task.version, action=self._action, arguments=self._arguments) if 'result' in self._kwargs: result = self._kwargs['result'] - if isinstance(result, misc.Failure): + if isinstance(result, ft.Failure): request['result'] = ('failure', result.to_dict()) else: request['result'] = ('success', result) @@ -417,7 +417,7 @@ class Response(Message): state = data['state'] data = data['data'] if state == FAILURE and 'result' in data: - data['result'] = misc.Failure.from_dict(data['result']) + data['result'] = ft.Failure.from_dict(data['result']) return cls(state, **data) @property diff --git a/taskflow/engines/worker_based/server.py b/taskflow/engines/worker_based/server.py index 9440c96a..db61edc6 100644 --- a/taskflow/engines/worker_based/server.py +++ b/taskflow/engines/worker_based/server.py @@ -21,6 +21,7 @@ import six from taskflow.engines.worker_based import protocol as pr from taskflow.engines.worker_based import proxy +from taskflow.types import failure as ft from taskflow.utils import misc LOG = logging.getLogger(__name__) @@ -69,20 +70,20 @@ class Server(object): failures=None, **kwargs): """Parse request before it can be further processed. - All `misc.Failure` objects that have been converted to dict on the - remote side will now converted back to `misc.Failure` objects. + All `failure.Failure` objects that have been converted to dict on the + remote side will now converted back to `failure.Failure` objects. """ action_args = dict(arguments=arguments, task_name=task_name) if result is not None: data_type, data = result if data_type == 'failure': - action_args['result'] = misc.Failure.from_dict(data) + action_args['result'] = ft.Failure.from_dict(data) else: action_args['result'] = data if failures is not None: action_args['failures'] = {} - for k, v in failures.items(): - action_args['failures'][k] = misc.Failure.from_dict(v) + for key, data in six.iteritems(failures): + action_args['failures'][key] = ft.Failure.from_dict(data) return task_cls, action, action_args @staticmethod @@ -218,7 +219,7 @@ class Server(object): message.delivery_tag, exc_info=True) reply_callback(result=failure.to_dict()) else: - if isinstance(result, misc.Failure): + if isinstance(result, ft.Failure): reply_callback(result=result.to_dict()) else: reply_callback(state=pr.SUCCESS, result=result) diff --git a/taskflow/examples/wrapped_exception.py b/taskflow/examples/wrapped_exception.py index dff6b2b4..78b5ad06 100644 --- a/taskflow/examples/wrapped_exception.py +++ b/taskflow/examples/wrapped_exception.py @@ -33,7 +33,7 @@ from taskflow import exceptions from taskflow.patterns import unordered_flow as uf from taskflow import task from taskflow.tests import utils -from taskflow.utils import misc +from taskflow.types import failure import example_utils as eu # noqa @@ -96,15 +96,15 @@ def run(**store): engine='parallel') except exceptions.WrappedFailure as ex: unknown_failures = [] - for failure in ex: - if failure.check(FirstException): - print("Got FirstException: %s" % failure.exception_str) - elif failure.check(SecondException): - print("Got SecondException: %s" % failure.exception_str) + for a_failure in ex: + if a_failure.check(FirstException): + print("Got FirstException: %s" % a_failure.exception_str) + elif a_failure.check(SecondException): + print("Got SecondException: %s" % a_failure.exception_str) else: - print("Unknown failure: %s" % failure) - unknown_failures.append(failure) - misc.Failure.reraise_if_any(unknown_failures) + print("Unknown failure: %s" % a_failure) + unknown_failures.append(a_failure) + failure.Failure.reraise_if_any(unknown_failures) eu.print_wrapped("Raise and catch first exception only") diff --git a/taskflow/exceptions.py b/taskflow/exceptions.py index cdffe0f9..876a3e3b 100644 --- a/taskflow/exceptions.py +++ b/taskflow/exceptions.py @@ -178,8 +178,8 @@ class WrappedFailure(Exception): See the failure class documentation for a more comprehensive set of reasons why this object *may* be reraised instead of the original exception. - :param causes: the :py:class:`~taskflow.utils.misc.Failure` objects that - caused this this exception to be raised. + :param causes: the :py:class:`~taskflow.types.failure.Failure` objects + that caused this this exception to be raised. """ def __init__(self, causes): diff --git a/taskflow/listeners/base.py b/taskflow/listeners/base.py index ca2ee6e2..e1d475f7 100644 --- a/taskflow/listeners/base.py +++ b/taskflow/listeners/base.py @@ -23,8 +23,8 @@ from oslo.utils import excutils import six from taskflow import states +from taskflow.types import failure from taskflow.types import notifier -from taskflow.utils import misc LOG = logging.getLogger(__name__) @@ -142,7 +142,7 @@ class LoggingBase(ListenerBase): result = details.get('result') exc_info = None was_failure = False - if isinstance(result, misc.Failure): + if isinstance(result, failure.Failure): if result.exc_info: exc_info = tuple(result.exc_info) was_failure = True diff --git a/taskflow/listeners/logging.py b/taskflow/listeners/logging.py index 87528f37..3629bb2c 100644 --- a/taskflow/listeners/logging.py +++ b/taskflow/listeners/logging.py @@ -21,8 +21,8 @@ import sys from taskflow.listeners import base from taskflow import states +from taskflow.types import failure from taskflow.types import notifier -from taskflow.utils import misc LOG = logging.getLogger(__name__) @@ -92,8 +92,8 @@ class DynamicLoggingListener(base.ListenerBase): * ``states.RETRYING`` * ``states.REVERTING`` - When a task produces a :py:class:`~taskflow.utils.misc.Failure` object as - its result (typically this happens when a task raises an exception) this + When a task produces a :py:class:`~taskflow.types.failure.Failure` object + as its result (typically this happens when a task raises an exception) this will **always** switch the logger to use ``logging.WARNING`` (if the failure object contains a ``exc_info`` tuple this will also be logged to provide a meaningful traceback). @@ -130,7 +130,7 @@ class DynamicLoggingListener(base.ListenerBase): # If the task failed, it's useful to show the exception traceback # and any other available exception information. result = details.get('result') - if isinstance(result, misc.Failure): + if isinstance(result, failure.Failure): if result.exc_info: exc_info = result.exc_info manual_tb = '' diff --git a/taskflow/persistence/backends/impl_sqlalchemy.py b/taskflow/persistence/backends/impl_sqlalchemy.py index 29ab8c97..4b12b782 100644 --- a/taskflow/persistence/backends/impl_sqlalchemy.py +++ b/taskflow/persistence/backends/impl_sqlalchemy.py @@ -37,6 +37,7 @@ from taskflow.persistence.backends import base from taskflow.persistence.backends.sqlalchemy import migration from taskflow.persistence.backends.sqlalchemy import models from taskflow.persistence import logbook +from taskflow.types import failure from taskflow.utils import async_utils from taskflow.utils import misc @@ -328,7 +329,7 @@ class Connection(base.Connection): pass except sa_exc.OperationalError as ex: if _is_db_connection_error(six.text_type(ex.args[0])): - failures.append(misc.Failure()) + failures.append(failure.Failure()) return False return True diff --git a/taskflow/persistence/logbook.py b/taskflow/persistence/logbook.py index 3d60aa52..f66f68ef 100644 --- a/taskflow/persistence/logbook.py +++ b/taskflow/persistence/logbook.py @@ -25,7 +25,7 @@ import six from taskflow import exceptions as exc from taskflow.openstack.common import uuidutils from taskflow import states -from taskflow.utils import misc +from taskflow.types import failure as ft LOG = logging.getLogger(__name__) @@ -50,7 +50,7 @@ def _safe_unmarshal_time(when): def _was_failure(state, result): - return state == states.FAILURE and isinstance(result, misc.Failure) + return state == states.FAILURE and isinstance(result, ft.Failure) def _fix_meta(data): @@ -363,7 +363,7 @@ class AtomDetail(object): self.meta = _fix_meta(data) failure = data.get('failure') if failure: - self.failure = misc.Failure.from_dict(failure) + self.failure = ft.Failure.from_dict(failure) @property def uuid(self): @@ -467,8 +467,8 @@ class RetryDetail(AtomDetail): new_results = [] for (data, failures) in results: new_failures = {} - for (key, failure_data) in six.iteritems(failures): - new_failures[key] = misc.Failure.from_dict(failure_data) + for (key, data) in six.iteritems(failures): + new_failures[key] = ft.Failure.from_dict(data) new_results.append((data, new_failures)) return new_results diff --git a/taskflow/storage.py b/taskflow/storage.py index 6ee10f60..c667509b 100644 --- a/taskflow/storage.py +++ b/taskflow/storage.py @@ -26,6 +26,7 @@ from taskflow.persistence import logbook from taskflow import retry from taskflow import states from taskflow import task +from taskflow.types import failure from taskflow.utils import lock_utils from taskflow.utils import misc from taskflow.utils import reflection @@ -425,7 +426,7 @@ class Storage(object): with self._lock.write_lock(): ad = self._atomdetail_by_name(atom_name) ad.put(state, data) - if state == states.FAILURE and isinstance(data, misc.Failure): + if state == states.FAILURE and isinstance(data, failure.Failure): # NOTE(imelnikov): failure serialization looses information, # so we cache failures here, in atom name -> failure mapping. self._failures[ad.name] = data diff --git a/taskflow/tests/unit/persistence/base.py b/taskflow/tests/unit/persistence/base.py index 3d28695c..6d96df66 100644 --- a/taskflow/tests/unit/persistence/base.py +++ b/taskflow/tests/unit/persistence/base.py @@ -20,7 +20,7 @@ from taskflow import exceptions as exc from taskflow.openstack.common import uuidutils from taskflow.persistence import logbook from taskflow import states -from taskflow.utils import misc +from taskflow.types import failure class PersistenceTestMixin(object): @@ -147,7 +147,7 @@ class PersistenceTestMixin(object): try: raise RuntimeError('Woot!') except Exception: - td.failure = misc.Failure() + td.failure = failure.Failure() fd.add(td) @@ -161,10 +161,9 @@ class PersistenceTestMixin(object): lb2 = conn.get_logbook(lb_id) fd2 = lb2.find(fd.uuid) td2 = fd2.find(td.uuid) - failure = td2.failure - self.assertEqual(failure.exception_str, 'Woot!') - self.assertIs(failure.check(RuntimeError), RuntimeError) - self.assertEqual(failure.traceback_str, td.failure.traceback_str) + self.assertEqual(td2.failure.exception_str, 'Woot!') + self.assertIs(td2.failure.check(RuntimeError), RuntimeError) + self.assertEqual(td2.failure.traceback_str, td.failure.traceback_str) self.assertIsInstance(td2, logbook.TaskDetail) def test_logbook_merge_flow_detail(self): @@ -269,7 +268,7 @@ class PersistenceTestMixin(object): fd = logbook.FlowDetail('test', uuid=uuidutils.generate_uuid()) lb.add(fd) rd = logbook.RetryDetail("retry-1", uuid=uuidutils.generate_uuid()) - fail = misc.Failure.from_exception(RuntimeError('fail')) + fail = failure.Failure.from_exception(RuntimeError('fail')) rd.results.append((42, {'some-task': fail})) fd.add(rd) @@ -286,7 +285,7 @@ class PersistenceTestMixin(object): rd2 = fd2.find(rd.uuid) self.assertIsInstance(rd2, logbook.RetryDetail) fail2 = rd2.results[0][1].get('some-task') - self.assertIsInstance(fail2, misc.Failure) + self.assertIsInstance(fail2, failure.Failure) self.assertTrue(fail.matches(fail2)) def test_retry_detail_save_intention(self): diff --git a/taskflow/tests/unit/test_engines.py b/taskflow/tests/unit/test_engines.py index 0823002d..d1c758b4 100644 --- a/taskflow/tests/unit/test_engines.py +++ b/taskflow/tests/unit/test_engines.py @@ -32,10 +32,10 @@ from taskflow import states from taskflow import task from taskflow import test from taskflow.tests import utils +from taskflow.types import failure from taskflow.types import futures from taskflow.types import graph as gr from taskflow.utils import async_utils as au -from taskflow.utils import misc from taskflow.utils import persistence_utils as p_utils @@ -529,7 +529,7 @@ class EngineCheckingTaskTest(utils.EngineTestBase): self.assertEqual(result, 'RESULT') self.assertEqual(list(flow_failures.keys()), ['fail1']) fail = flow_failures['fail1'] - self.assertIsInstance(fail, misc.Failure) + self.assertIsInstance(fail, failure.Failure) self.assertEqual(str(fail), 'Failure: RuntimeError: Woot!') flow = lf.Flow('test').add( diff --git a/taskflow/tests/unit/test_failure.py b/taskflow/tests/unit/test_failure.py index 3f4d001e..64e1c44d 100644 --- a/taskflow/tests/unit/test_failure.py +++ b/taskflow/tests/unit/test_failure.py @@ -22,7 +22,6 @@ from taskflow import exceptions from taskflow import test from taskflow.tests import utils as test_utils from taskflow.types import failure -from taskflow.utils import misc def _captured_failure(msg): @@ -217,7 +216,7 @@ class FailureObjectTestCase(test.TestCase): def test_pformat_traceback_captured_no_exc_info(self): captured = _captured_failure('Woot!') - captured = misc.Failure.from_dict(captured.to_dict()) + captured = failure.Failure.from_dict(captured.to_dict()) text = captured.pformat(traceback=True) self.assertIn("Traceback (most recent call last):", text) diff --git a/taskflow/tests/unit/test_storage.py b/taskflow/tests/unit/test_storage.py index 7d3b55b6..deb4db4f 100644 --- a/taskflow/tests/unit/test_storage.py +++ b/taskflow/tests/unit/test_storage.py @@ -25,7 +25,7 @@ from taskflow import states from taskflow import storage from taskflow import test from taskflow.tests import utils as test_utils -from taskflow.utils import misc +from taskflow.types import failure from taskflow.utils import persistence_utils as p_utils @@ -128,46 +128,46 @@ class StorageTestMixin(object): self.assertEqual(s.get_atom_state('my task'), states.FAILURE) def test_save_and_get_cached_failure(self): - failure = misc.Failure.from_exception(RuntimeError('Woot!')) + a_failure = failure.Failure.from_exception(RuntimeError('Woot!')) s = self._get_storage() s.ensure_atom(test_utils.NoopTask('my task')) - s.save('my task', failure, states.FAILURE) - self.assertEqual(s.get('my task'), failure) + s.save('my task', a_failure, states.FAILURE) + self.assertEqual(s.get('my task'), a_failure) self.assertEqual(s.get_atom_state('my task'), states.FAILURE) self.assertTrue(s.has_failures()) - self.assertEqual(s.get_failures(), {'my task': failure}) + self.assertEqual(s.get_failures(), {'my task': a_failure}) def test_save_and_get_non_cached_failure(self): - failure = misc.Failure.from_exception(RuntimeError('Woot!')) + a_failure = failure.Failure.from_exception(RuntimeError('Woot!')) s = self._get_storage() s.ensure_atom(test_utils.NoopTask('my task')) - s.save('my task', failure, states.FAILURE) - self.assertEqual(s.get('my task'), failure) + s.save('my task', a_failure, states.FAILURE) + self.assertEqual(s.get('my task'), a_failure) s._failures['my task'] = None - self.assertTrue(failure.matches(s.get('my task'))) + self.assertTrue(a_failure.matches(s.get('my task'))) def test_get_failure_from_reverted_task(self): - failure = misc.Failure.from_exception(RuntimeError('Woot!')) + a_failure = failure.Failure.from_exception(RuntimeError('Woot!')) s = self._get_storage() s.ensure_atom(test_utils.NoopTask('my task')) - s.save('my task', failure, states.FAILURE) + s.save('my task', a_failure, states.FAILURE) s.set_atom_state('my task', states.REVERTING) - self.assertEqual(s.get('my task'), failure) + self.assertEqual(s.get('my task'), a_failure) s.set_atom_state('my task', states.REVERTED) - self.assertEqual(s.get('my task'), failure) + self.assertEqual(s.get('my task'), a_failure) def test_get_failure_after_reload(self): - failure = misc.Failure.from_exception(RuntimeError('Woot!')) + a_failure = failure.Failure.from_exception(RuntimeError('Woot!')) s = self._get_storage() s.ensure_atom(test_utils.NoopTask('my task')) - s.save('my task', failure, states.FAILURE) + s.save('my task', a_failure, states.FAILURE) s2 = self._get_storage(s._flowdetail) self.assertTrue(s2.has_failures()) self.assertEqual(1, len(s2.get_failures())) - self.assertTrue(failure.matches(s2.get('my task'))) + self.assertTrue(a_failure.matches(s2.get('my task'))) self.assertEqual(s2.get_atom_state('my task'), states.FAILURE) def test_get_non_existing_var(self): @@ -486,15 +486,15 @@ class StorageTestMixin(object): self.assertEqual(s.fetch_all(), {}) def test_cached_retry_failure(self): - failure = misc.Failure.from_exception(RuntimeError('Woot!')) + a_failure = failure.Failure.from_exception(RuntimeError('Woot!')) s = self._get_storage() s.ensure_atom(test_utils.NoopRetry('my retry', provides=['x'])) s.save('my retry', 'a') - s.save('my retry', failure, states.FAILURE) + s.save('my retry', a_failure, states.FAILURE) history = s.get_retry_history('my retry') - self.assertEqual(history, [('a', {}), (failure, {})]) + self.assertEqual(history, [('a', {}), (a_failure, {})]) self.assertIs(s.has_failures(), True) - self.assertEqual(s.get_failures(), {'my retry': failure}) + self.assertEqual(s.get_failures(), {'my retry': a_failure}) def test_logbook_get_unknown_atom_type(self): self.assertRaisesRegexp(TypeError, diff --git a/taskflow/tests/unit/worker_based/test_executor.py b/taskflow/tests/unit/worker_based/test_executor.py index 3e494e88..d2b97bfe 100644 --- a/taskflow/tests/unit/worker_based/test_executor.py +++ b/taskflow/tests/unit/worker_based/test_executor.py @@ -24,7 +24,7 @@ from taskflow.engines.worker_based import protocol as pr from taskflow import test from taskflow.test import mock from taskflow.tests import utils as test_utils -from taskflow.utils import misc +from taskflow.types import failure from taskflow.utils import threading_utils @@ -111,8 +111,8 @@ class TestWorkerTaskExecutor(test.MockTestCase): [mock.call.on_progress(progress=1.0)]) def test_on_message_response_state_failure(self): - failure = misc.Failure.from_exception(Exception('test')) - failure_dict = failure.to_dict() + a_failure = failure.Failure.from_exception(Exception('test')) + failure_dict = a_failure.to_dict() response = pr.Response(pr.FAILURE, result=failure_dict) ex = self.executor() ex._requests_cache[self.task_uuid] = self.request_inst_mock @@ -121,7 +121,7 @@ class TestWorkerTaskExecutor(test.MockTestCase): self.assertEqual(len(ex._requests_cache), 0) expected_calls = [ mock.call.transition_and_log_error(pr.FAILURE, logger=mock.ANY), - mock.call.set_result(result=test_utils.FailureMatcher(failure)) + mock.call.set_result(result=test_utils.FailureMatcher(a_failure)) ] self.assertEqual(expected_calls, self.request_inst_mock.mock_calls) diff --git a/taskflow/tests/unit/worker_based/test_pipeline.py b/taskflow/tests/unit/worker_based/test_pipeline.py index ae11efd2..b86cedd0 100644 --- a/taskflow/tests/unit/worker_based/test_pipeline.py +++ b/taskflow/tests/unit/worker_based/test_pipeline.py @@ -24,7 +24,7 @@ from taskflow.engines.worker_based import server as worker_server from taskflow.openstack.common import uuidutils from taskflow import test from taskflow.tests import utils as test_utils -from taskflow.utils import misc +from taskflow.types import failure TEST_EXCHANGE, TEST_TOPIC = ('test-exchange', 'test-topic') @@ -94,5 +94,5 @@ class TestPipeline(test.TestCase): executor.wait_for_any([f]) _t2, _action, result = f.result() - self.assertIsInstance(result, misc.Failure) + self.assertIsInstance(result, failure.Failure) self.assertEqual(RuntimeError, result.check(RuntimeError)) diff --git a/taskflow/tests/unit/worker_based/test_protocol.py b/taskflow/tests/unit/worker_based/test_protocol.py index 6df2bb41..4c9c4b77 100644 --- a/taskflow/tests/unit/worker_based/test_protocol.py +++ b/taskflow/tests/unit/worker_based/test_protocol.py @@ -23,7 +23,7 @@ from taskflow.openstack.common import uuidutils from taskflow import test from taskflow.test import mock from taskflow.tests import utils -from taskflow.utils import misc +from taskflow.types import failure class TestProtocolValidation(test.TestCase): @@ -149,15 +149,16 @@ class TestProtocol(test.TestCase): self.request_to_dict(result=('success', None))) def test_to_dict_with_result_failure(self): - failure = misc.Failure.from_exception(RuntimeError('Woot!')) - expected = self.request_to_dict(result=('failure', failure.to_dict())) - self.assertEqual(self.request(result=failure).to_dict(), expected) + a_failure = failure.Failure.from_exception(RuntimeError('Woot!')) + expected = self.request_to_dict(result=('failure', + a_failure.to_dict())) + self.assertEqual(self.request(result=a_failure).to_dict(), expected) def test_to_dict_with_failures(self): - failure = misc.Failure.from_exception(RuntimeError('Woot!')) - request = self.request(failures={self.task.name: failure}) + a_failure = failure.Failure.from_exception(RuntimeError('Woot!')) + request = self.request(failures={self.task.name: a_failure}) expected = self.request_to_dict( - failures={self.task.name: failure.to_dict()}) + failures={self.task.name: a_failure.to_dict()}) self.assertEqual(request.to_dict(), expected) def test_pending_not_expired(self): diff --git a/taskflow/tests/unit/worker_based/test_server.py b/taskflow/tests/unit/worker_based/test_server.py index b6e62671..5e9129ae 100644 --- a/taskflow/tests/unit/worker_based/test_server.py +++ b/taskflow/tests/unit/worker_based/test_server.py @@ -22,7 +22,7 @@ from taskflow.engines.worker_based import server from taskflow import test from taskflow.test import mock from taskflow.tests import utils -from taskflow.utils import misc +from taskflow.types import failure class TestServer(test.MockTestCase): @@ -121,19 +121,19 @@ class TestServer(test.MockTestCase): result=1))) def test_parse_request_with_failure_result(self): - failure = misc.Failure.from_exception(Exception('test')) - request = self.make_request(action='revert', result=failure) + a_failure = failure.Failure.from_exception(Exception('test')) + request = self.make_request(action='revert', result=a_failure) task_cls, action, task_args = server.Server._parse_request(**request) self.assertEqual((task_cls, action, task_args), (self.task.name, 'revert', dict(task_name=self.task.name, arguments=self.task_args, - result=utils.FailureMatcher(failure)))) + result=utils.FailureMatcher(a_failure)))) def test_parse_request_with_failures(self): - failures = {'0': misc.Failure.from_exception(Exception('test1')), - '1': misc.Failure.from_exception(Exception('test2'))} + failures = {'0': failure.Failure.from_exception(Exception('test1')), + '1': failure.Failure.from_exception(Exception('test2'))} request = self.make_request(action='revert', failures=failures) task_cls, action, task_args = server.Server._parse_request(**request) @@ -220,16 +220,16 @@ class TestServer(test.MockTestCase): self.assertEqual(self.master_mock.mock_calls, []) self.assertTrue(mocked_exception.called) - @mock.patch.object(misc.Failure, 'from_dict') - @mock.patch.object(misc.Failure, 'to_dict') + @mock.patch.object(failure.Failure, 'from_dict') + @mock.patch.object(failure.Failure, 'to_dict') def test_process_request_parse_request_failure(self, to_mock, from_mock): failure_dict = { 'failure': 'failure', } - failure = misc.Failure.from_exception(RuntimeError('Woot!')) + a_failure = failure.Failure.from_exception(RuntimeError('Woot!')) to_mock.return_value = failure_dict from_mock.side_effect = ValueError('Woot!') - request = self.make_request(result=failure) + request = self.make_request(result=a_failure) # create server and process request s = self.server(reset_master_mock=True) @@ -244,7 +244,7 @@ class TestServer(test.MockTestCase): ] self.assertEqual(master_mock_calls, self.master_mock.mock_calls) - @mock.patch.object(misc.Failure, 'to_dict') + @mock.patch.object(failure.Failure, 'to_dict') def test_process_request_endpoint_not_found(self, to_mock): failure_dict = { 'failure': 'failure', @@ -265,7 +265,7 @@ class TestServer(test.MockTestCase): ] self.assertEqual(self.master_mock.mock_calls, master_mock_calls) - @mock.patch.object(misc.Failure, 'to_dict') + @mock.patch.object(failure.Failure, 'to_dict') def test_process_request_execution_failure(self, to_mock): failure_dict = { 'failure': 'failure', @@ -287,7 +287,7 @@ class TestServer(test.MockTestCase): ] self.assertEqual(self.master_mock.mock_calls, master_mock_calls) - @mock.patch.object(misc.Failure, 'to_dict') + @mock.patch.object(failure.Failure, 'to_dict') def test_process_request_task_failure(self, to_mock): failure_dict = { 'failure': 'failure', diff --git a/taskflow/tests/utils.py b/taskflow/tests/utils.py index d01f91a3..a0b2ff0d 100644 --- a/taskflow/tests/utils.py +++ b/taskflow/tests/utils.py @@ -23,8 +23,8 @@ from taskflow import exceptions from taskflow.persistence.backends import impl_memory from taskflow import retry from taskflow import task +from taskflow.types import failure from taskflow.utils import kazoo_utils -from taskflow.utils import misc from taskflow.utils import threading_utils ARGS_KEY = '__args__' @@ -50,7 +50,7 @@ def wrap_all_failures(): try: yield except Exception: - raise exceptions.WrappedFailure([misc.Failure()]) + raise exceptions.WrappedFailure([failure.Failure()]) def zookeeper_available(min_version, timeout=3): diff --git a/taskflow/types/failure.py b/taskflow/types/failure.py index 93829ad6..c1f28be8 100644 --- a/taskflow/types/failure.py +++ b/taskflow/types/failure.py @@ -68,7 +68,7 @@ class Failure(object): exception and desire to reraise it to the user/caller of the WBE based engine for appropriate handling (this matches the behavior of non-remote engines). To accomplish this a failure object (or a - :py:meth:`~misc.Failure.to_dict` form) would be sent over the WBE channel + :py:meth:`~.Failure.to_dict` form) would be sent over the WBE channel and the WBE based engine would deserialize it and use this objects :meth:`.reraise` method to cause an exception that contains similar/equivalent information as the original exception to be reraised, From b3134c96835f111f187fa24462f8fa69de44b0d8 Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Wed, 22 Oct 2014 19:17:49 +0000 Subject: [PATCH 084/240] Updated from global requirements Change-Id: I39c4858101f83a269a3a0b2f1c49497eb4768001 --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 7d97d4ea..d75489dc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,7 +3,7 @@ # process, which may cause wedges in the gate later. hacking>=0.9.2,<0.10 -oslotest>=1.1.0 # Apache-2.0 +oslotest>=1.2.0 # Apache-2.0 mock>=1.0 testtools>=0.9.34 From 5671868338327cbd680f7f7b52ac9bfc57e18236 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 24 Oct 2014 09:31:00 -0400 Subject: [PATCH 085/240] Add pbr to installation requirements Add pbr to the list of installation requirements so that it is installed via pip before this library is installed, instead of with easy_install. This avoids issues like Bug #1384919, and ensures that projects that use this library as a dependency are properly installed. Change-Id: I6c155370dbd01fe4748d5137bdf288e8d3e1a67e --- requirements-py2.txt | 2 ++ requirements-py3.txt | 2 ++ 2 files changed, 4 insertions(+) diff --git a/requirements-py2.txt b/requirements-py2.txt index 61348779..260f535e 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -2,6 +2,8 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. +pbr>=0.6,!=0.7,<1.0 + # Packages needed for using this library. # Only needed on python 2.6 diff --git a/requirements-py3.txt b/requirements-py3.txt index ea305822..446e5798 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -2,6 +2,8 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. +pbr>=0.6,!=0.7,<1.0 + # Packages needed for using this library. # Python 2->3 compatibility library. From edb9212162f5069db33747c3421e3345fed1ccdc Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 24 Oct 2014 22:16:55 -0700 Subject: [PATCH 086/240] Handle the case where '_exc_type_names' is empty If '_exc_type_names' is empty we want to avoid having a index error or type error and want to handle just not writing out the first exception type name and instead just write out the exception text string. Change-Id: I29fd740fd7052f73975d84ec03455170c420ca89 --- taskflow/tests/unit/test_failure.py | 8 ++++++++ taskflow/types/failure.py | 7 +++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/taskflow/tests/unit/test_failure.py b/taskflow/tests/unit/test_failure.py index 3f4d001e..a7c7e68f 100644 --- a/taskflow/tests/unit/test_failure.py +++ b/taskflow/tests/unit/test_failure.py @@ -114,6 +114,14 @@ class ReCreatedFailureTestCase(test.TestCase, GeneralFailureObjTestsMixin): self.fail_obj.reraise) self.assertIs(exc.check(RuntimeError), RuntimeError) + def test_no_type_names(self): + fail_obj = _captured_failure('Woot!') + fail_obj = failure.Failure(exception_str=fail_obj.exception_str, + traceback_str=fail_obj.traceback_str, + exc_type_names=[]) + self.assertEqual([], list(fail_obj)) + self.assertEqual("Failure: Woot!", fail_obj.pformat()) + class FromExceptionTestCase(test.TestCase, GeneralFailureObjTestsMixin): diff --git a/taskflow/types/failure.py b/taskflow/types/failure.py index 93829ad6..17e2cdc2 100644 --- a/taskflow/types/failure.py +++ b/taskflow/types/failure.py @@ -241,8 +241,11 @@ class Failure(object): def pformat(self, traceback=False): """Pretty formats the failure object into a string.""" buf = six.StringIO() - buf.write( - 'Failure: %s: %s' % (self._exc_type_names[0], self._exception_str)) + if not self._exc_type_names: + buf.write('Failure: %s' % (self._exception_str)) + else: + buf.write('Failure: %s: %s' % (self._exc_type_names[0], + self._exception_str)) if traceback: if self._traceback_str is not None: traceback_str = self._traceback_str.rstrip() From 34b358a137b307eb11bb362baa4459613a56b3e0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 25 Oct 2014 21:58:48 -0700 Subject: [PATCH 087/240] Use standard threading locks in the cache types Instead of using a reader/writer lock in the cache types just take advantage of the fact that single, non-mutating operations on dictionaries are thread-safe in python. This means we can remove the need to have simple read-only operations using read-locks and can just use a simpler lock around write-operations or around read/write operations that span multiple dictionary operations (like iteration). Change-Id: I4679275d7fe25fac12a6b05e8aa38da95649f4f6 --- taskflow/engines/worker_based/cache.py | 4 ++-- taskflow/types/cache.py | 22 ++++++++++------------ 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/taskflow/engines/worker_based/cache.py b/taskflow/engines/worker_based/cache.py index 9da7f12c..3b00890d 100644 --- a/taskflow/engines/worker_based/cache.py +++ b/taskflow/engines/worker_based/cache.py @@ -28,7 +28,7 @@ class RequestsCache(base.ExpiringCache): def get_waiting_requests(self, tasks): """Get list of waiting requests by tasks.""" waiting_requests = [] - with self._lock.read_lock(): + with self._lock: for request in six.itervalues(self._data): if request.state == pr.WAITING and request.task_cls in tasks: waiting_requests.append(request) @@ -41,7 +41,7 @@ class WorkersCache(base.ExpiringCache): def get_topic_by_task(self, task): """Get topic for a given task.""" available_topics = [] - with self._lock.read_lock(): + with self._lock: for topic, tasks in six.iteritems(self._data): if task in tasks: available_topics.append(topic) diff --git a/taskflow/types/cache.py b/taskflow/types/cache.py index 72214fed..61511e12 100644 --- a/taskflow/types/cache.py +++ b/taskflow/types/cache.py @@ -14,9 +14,10 @@ # License for the specific language governing permissions and limitations # under the License. +import threading + import six -from taskflow.utils import lock_utils as lu from taskflow.utils import reflection @@ -30,41 +31,38 @@ class ExpiringCache(object): def __init__(self): self._data = {} - self._lock = lu.ReaderWriterLock() + self._lock = threading.Lock() def __setitem__(self, key, value): """Set a value in the cache.""" - with self._lock.write_lock(): + with self._lock: self._data[key] = value def __len__(self): """Returns how many items are in this cache.""" - with self._lock.read_lock(): - return len(self._data) + return len(self._data) def get(self, key, default=None): """Retrieve a value from the cache (returns default if not found).""" - with self._lock.read_lock(): - return self._data.get(key, default) + return self._data.get(key, default) def __getitem__(self, key): """Retrieve a value from the cache.""" - with self._lock.read_lock(): - return self._data[key] + return self._data[key] def __delitem__(self, key): """Delete a key & value from the cache.""" - with self._lock.write_lock(): + with self._lock: del self._data[key] def cleanup(self, on_expired_callback=None): """Delete out-dated keys & values from the cache.""" - with self._lock.write_lock(): + with self._lock: expired_values = [(k, v) for k, v in six.iteritems(self._data) if v.expired] for (k, _v) in expired_values: del self._data[k] - if on_expired_callback: + if on_expired_callback is not None: arg_c = len(reflection.get_callable_args(on_expired_callback)) for (k, v) in expired_values: if arg_c == 2: From 59f45e8b145ceb2f0fd962be5a3eb6e4567ee335 Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Mon, 27 Oct 2014 12:22:10 +0000 Subject: [PATCH 088/240] Updated from global requirements Change-Id: I5d28190e7687d331b6c01cc66adc9f4793178c96 --- requirements-py2.txt | 2 +- requirements-py3.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-py2.txt b/requirements-py2.txt index 260f535e..fe897b7b 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -16,7 +16,7 @@ six>=1.7.0 networkx>=1.8 # Used for backend storage engine loading. -stevedore>=1.0.0 # Apache-2.0 +stevedore>=1.1.0 # Apache-2.0 # Backport for concurrent.futures which exists in 3.2+ futures>=2.1.6 diff --git a/requirements-py3.txt b/requirements-py3.txt index 446e5798..f135eff4 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -13,7 +13,7 @@ six>=1.7.0 networkx>=1.8 # Used for backend storage engine loading. -stevedore>=1.0.0 # Apache-2.0 +stevedore>=1.1.0 # Apache-2.0 # Used for structured input validation jsonschema>=2.0.0,<3.0.0 From 543b6a06f7dc64b6ef604d52cb13d61cf9bc1f63 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 29 Oct 2014 14:42:49 -0700 Subject: [PATCH 089/240] Link bug in requirements so people understand why pbr is listed To make it more obvious why pbr is the first requirement in the file/s add a link to the bug that caused this requirement to be re-added so that for future prosperity/generations... it is known. Change-Id: I11b23675cc379934e83b237bc30ca6baa0a87751 --- requirements-py2.txt | 1 + requirements-py3.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements-py2.txt b/requirements-py2.txt index fe897b7b..0827e7ed 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -2,6 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. +# See: https://bugs.launchpad.net/pbr/+bug/1384919 for why this is here... pbr>=0.6,!=0.7,<1.0 # Packages needed for using this library. diff --git a/requirements-py3.txt b/requirements-py3.txt index f135eff4..5f265dcc 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -2,6 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. +# See: https://bugs.launchpad.net/pbr/+bug/1384919 for why this is here... pbr>=0.6,!=0.7,<1.0 # Packages needed for using this library. From c8b0f25d816bfd00a93134567cb5e0f313bfe9f2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 7 Oct 2014 15:55:54 -0700 Subject: [PATCH 090/240] Reduce the worker-engine joint testing time This adjustment tweaks how workers and an engine use there configuration (making it so they share the majority of configuration). This also allows both engine and worker to share the same configuration, which allows them to share the same polling interval and so-on. This has the effect of reducing the test time on a local machine that I am running them on (one of many that I run them on). Before: Ran 1022 tests in ~130s (same machine) After: Ran 1022 tests in ~60s (same machine) Change-Id: I2150d26fb52cb86d61682124d710ba15c7ff27a6 --- taskflow/tests/unit/test_engines.py | 53 +++++++++++++++++------------ 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/taskflow/tests/unit/test_engines.py b/taskflow/tests/unit/test_engines.py index 4ce291d1..c6d11e31 100644 --- a/taskflow/tests/unit/test_engines.py +++ b/taskflow/tests/unit/test_engines.py @@ -15,7 +15,6 @@ # under the License. import contextlib -import threading from concurrent import futures import testtools @@ -37,6 +36,7 @@ from taskflow.types import graph as gr from taskflow.utils import eventlet_utils as eu from taskflow.utils import misc from taskflow.utils import persistence_utils as p_utils +from taskflow.utils import threading_utils as tu class EngineTaskTest(utils.EngineTestBase): @@ -613,28 +613,41 @@ class WorkerBasedEngineTest(EngineTaskTest, EngineLinearAndUnorderedExceptionsTest, EngineGraphFlowTest, test.TestCase): - def setUp(self): super(WorkerBasedEngineTest, self).setUp() - self.exchange = 'test' - self.topic = 'topic' - self.transport = 'memory' - worker_conf = { - 'exchange': self.exchange, - 'topic': self.topic, - 'tasks': [ - 'taskflow.tests.utils', - ], - 'transport': self.transport, + shared_conf = { + 'exchange': 'test', + 'transport': 'memory', 'transport_options': { - 'polling_interval': 0.01 - } + # NOTE(imelnikov): I run tests several times for different + # intervals. Reducing polling interval below 0.01 did not give + # considerable win in tests run time; reducing polling interval + # too much (AFAIR below 0.0005) affected stability -- I was + # seeing timeouts. So, 0.01 looks like the most balanced for + # local transports (for now). + 'polling_interval': 0.01, + }, } + worker_conf = shared_conf.copy() + worker_conf.update({ + 'topic': 'my-topic', + 'tasks': [ + # This makes it possible for the worker to run/find any atoms + # that are defined in the test.utils module (which are all + # the task/atom types that this test uses)... + utils.__name__, + ], + }) + self.engine_conf = shared_conf.copy() + self.engine_conf.update({ + 'engine': 'worker-based', + 'topics': tuple([worker_conf['topic']]), + }) self.worker = wkr.Worker(**worker_conf) - self.worker_thread = threading.Thread(target=self.worker.run) - self.worker_thread.daemon = True + self.worker_thread = tu.daemon_thread(target=self.worker.run) self.worker_thread.start() - # make sure worker is started before we can continue + + # Make sure the worker is started before we can continue... self.worker.wait() def tearDown(self): @@ -644,11 +657,7 @@ class WorkerBasedEngineTest(EngineTaskTest, def _make_engine(self, flow, flow_detail=None): return taskflow.engines.load(flow, flow_detail=flow_detail, - backend=self.backend, - engine='worker-based', - exchange=self.exchange, - topics=[self.topic], - transport=self.transport) + backend=self.backend, **self.engine_conf) def test_correct_load(self): engine = self._make_engine(utils.TaskNoRequiresNoReturns) From 74ebb43474e3ec2a5b9f832f2a44b67b8c83dc68 Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Wed, 5 Nov 2014 08:36:17 +0000 Subject: [PATCH 091/240] Updated from global requirements Change-Id: Ida3d96383c54ff6b16297f5bfc46804dc954bc59 --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index d75489dc..fe1848c5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,7 +5,7 @@ hacking>=0.9.2,<0.10 oslotest>=1.2.0 # Apache-2.0 mock>=1.0 -testtools>=0.9.34 +testtools>=0.9.36 # Used for testing the WBE engine. kombu>=2.5.0 From e68e1f8d1c3f855aa80856003e5df1f328faf8d5 Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Sun, 16 Nov 2014 15:07:35 +0000 Subject: [PATCH 092/240] Updated from global requirements Change-Id: I027659520dad2166b8e28ec0a2334cd58c563dfc --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index fe1848c5..36857441 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,7 +5,7 @@ hacking>=0.9.2,<0.10 oslotest>=1.2.0 # Apache-2.0 mock>=1.0 -testtools>=0.9.36 +testtools>=0.9.36,!=1.2.0 # Used for testing the WBE engine. kombu>=2.5.0 From afe2a9339c02dd1607f440a72f813d28c5a18685 Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Tue, 18 Nov 2014 11:37:36 +0000 Subject: [PATCH 093/240] Updated from global requirements Change-Id: I3af87566b255957c273a8b9f392425877394ce8a --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 36857441..3a669cb4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,7 +5,7 @@ hacking>=0.9.2,<0.10 oslotest>=1.2.0 # Apache-2.0 mock>=1.0 -testtools>=0.9.36,!=1.2.0 +testtools>=0.9.36,!=1.2.0,!=1.4.0 # Used for testing the WBE engine. kombu>=2.5.0 From f3e4bb08631403df83d79a57ea5e71bfc18a2d0e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 19 Oct 2014 19:52:03 -0700 Subject: [PATCH 094/240] Use wrapt to provide the deprecated class proxy Instead of having our own mini-proxy class use the robustness (and correctness) that the wrapt library provides to implement a more compliant object proxy that handles edge-cases better than our own. Also adds a few basic sanity tests to ensure that the moved/deprecated classes operate as we want them to. Change-Id: Ib7ca832700583d3ca5e175cb322aa00543cbc475 --- requirements-py2.txt | 3 ++ requirements-py3.txt | 3 ++ taskflow/tests/unit/test_failure.py | 12 +++++ taskflow/tests/unit/test_notifier.py | 9 ++++ taskflow/utils/deprecation.py | 75 ++++++---------------------- 5 files changed, 42 insertions(+), 60 deletions(-) diff --git a/requirements-py2.txt b/requirements-py2.txt index 0827e7ed..a1de6663 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -13,6 +13,9 @@ ordereddict # Python 2->3 compatibility library. six>=1.7.0 +# For proxying objects and making correct decorators +wrapt>=1.7.0 + # Very nice graph library networkx>=1.8 diff --git a/requirements-py3.txt b/requirements-py3.txt index 5f265dcc..549a5dfa 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -10,6 +10,9 @@ pbr>=0.6,!=0.7,<1.0 # Python 2->3 compatibility library. six>=1.7.0 +# For proxying objects and making correct decorators +wrapt>=1.7.0 + # Very nice graph library networkx>=1.8 diff --git a/taskflow/tests/unit/test_failure.py b/taskflow/tests/unit/test_failure.py index 793274e1..b70add4f 100644 --- a/taskflow/tests/unit/test_failure.py +++ b/taskflow/tests/unit/test_failure.py @@ -22,6 +22,7 @@ from taskflow import exceptions from taskflow import test from taskflow.tests import utils as test_utils from taskflow.types import failure +from taskflow.utils import misc def _captured_failure(msg): @@ -38,6 +39,17 @@ def _make_exc_info(msg): return sys.exc_info() +class DeprecatedTestCase(test.TestCase): + def test_deprecated(self): + f = None + try: + raise RuntimeError("Woot!") + except RuntimeError: + f = misc.Failure() + self.assertIsInstance(f, failure.Failure) + self.assertIsInstance(f, misc.Failure) + + class GeneralFailureObjTestsMixin(object): def test_captures_message(self): diff --git a/taskflow/tests/unit/test_notifier.py b/taskflow/tests/unit/test_notifier.py index 0761cb6e..61dc7746 100644 --- a/taskflow/tests/unit/test_notifier.py +++ b/taskflow/tests/unit/test_notifier.py @@ -20,6 +20,15 @@ import functools from taskflow import states from taskflow import test from taskflow.types import notifier as nt +from taskflow.utils import misc + + +class DeprecatedTestCase(test.TestCase): + def test_deprecated(self): + notifier = misc.Notifier() + self.assertIsInstance(notifier, misc.Notifier) + self.assertIsInstance(notifier, nt.Notifier) + self.assertTrue(hasattr(misc.Notifier, 'ANY')) class NotifierTest(test.TestCase): diff --git a/taskflow/utils/deprecation.py b/taskflow/utils/deprecation.py index 899e035d..b9a1c724 100644 --- a/taskflow/utils/deprecation.py +++ b/taskflow/utils/deprecation.py @@ -16,6 +16,8 @@ import warnings +import wrapt + from taskflow.utils import reflection @@ -24,72 +26,25 @@ def deprecation(message, stacklevel=2): warnings.warn(message, category=DeprecationWarning, stacklevel=stacklevel) -# Helper accessors for the moved proxy (since it will not have easy access -# to its own getattr and setattr functions). -_setattr = object.__setattr__ -_getattr = object.__getattribute__ - - -class MovedClassProxy(object): - """Acts as a proxy to a class that was moved to another location. - - Partially based on: - - http://code.activestate.com/recipes/496741-object-proxying/ and other - various examination of how to make a good enough proxy for our usage to - move the various types we want to move during the deprecation process. - - And partially based on the wrapt object proxy (which we should just use - when it becomes available @ http://review.openstack.org/#/c/94754/). - """ - - __slots__ = [ - '__wrapped__', '__message__', '__stacklevel__', - # Ensure weakrefs can be made, - # https://docs.python.org/2/reference/datamodel.html#slots - '__weakref__', - ] +class MovedClassProxy(wrapt.ObjectProxy): + """Acts as a proxy to a class that was moved to another location.""" def __init__(self, wrapped, message, stacklevel): - # We can't assign to these directly, since we are overriding getattr - # and setattr and delattr so we have to do this hoop jump to ensure - # that we don't invoke those methods (and cause infinite recursion). - _setattr(self, '__wrapped__', wrapped) - _setattr(self, '__message__', message) - _setattr(self, '__stacklevel__', stacklevel) - try: - _setattr(self, '__qualname__', wrapped.__qualname__) - except AttributeError: - pass - - def __instancecheck__(self, instance): - deprecation( - _getattr(self, '__message__'), _getattr(self, '__stacklevel__')) - return isinstance(instance, _getattr(self, '__wrapped__')) - - def __subclasscheck__(self, instance): - deprecation( - _getattr(self, '__message__'), _getattr(self, '__stacklevel__')) - return issubclass(instance, _getattr(self, '__wrapped__')) + super(MovedClassProxy, self).__init__(wrapped) + self._self_message = message + self._self_stacklevel = stacklevel def __call__(self, *args, **kwargs): - deprecation( - _getattr(self, '__message__'), _getattr(self, '__stacklevel__')) - return _getattr(self, '__wrapped__')(*args, **kwargs) + deprecation(self._self_message, stacklevel=self._self_stacklevel) + return self.__wrapped__(*args, **kwargs) - def __getattribute__(self, name): - return getattr(_getattr(self, '__wrapped__'), name) + def __instancecheck__(self, instance): + deprecation(self._self_message, stacklevel=self._self_stacklevel) + return isinstance(instance, self.__wrapped__) - def __setattr__(self, name, value): - setattr(_getattr(self, '__wrapped__'), name, value) - - def __delattr__(self, name): - delattr(_getattr(self, '__wrapped__'), name) - - def __repr__(self): - wrapped = _getattr(self, '__wrapped__') - return "<%s at 0x%x for %r at 0x%x>" % ( - type(self).__name__, id(self), wrapped, id(wrapped)) + def __subclasscheck__(self, instance): + deprecation(self._self_message, stacklevel=self._self_stacklevel) + return issubclass(instance, self.__wrapped__) def moved_class(new_class, old_class_name, old_module_name, message=None, From b24656c25859ad9530b80377c5315f4744cc468f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 27 Jun 2014 17:40:28 -0700 Subject: [PATCH 095/240] Mark 'task_notifier' as renamed to 'atom_notifier' Deprecate the usage of an engines 'task_notifier' in favor of the more appropriately named 'atom_notifier' and mark the 'task_notifier' property as subject to removal in a future version. This makes the usage of this notifier more clear since it is not only used for task notification but also for retry notification (which is why naming it atom notifier is more suited to its actual usage). Change-Id: Idfa31a6665e43dcc9360f0aed40e27817d9e6737 --- taskflow/engines/action_engine/engine.py | 2 +- taskflow/engines/action_engine/runtime.py | 8 +-- taskflow/engines/base.py | 26 +++++++++- taskflow/utils/deprecation.py | 62 +++++++++++++++++------ 4 files changed, 76 insertions(+), 22 deletions(-) diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index 730d2895..5a0fc9fb 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -217,7 +217,7 @@ class ActionEngine(base.EngineBase): self._compilation = self._compiler.compile() self._runtime = runtime.Runtime(self._compilation, self.storage, - self.task_notifier, + self.atom_notifier, self._task_executor) self._compiled = True diff --git a/taskflow/engines/action_engine/runtime.py b/taskflow/engines/action_engine/runtime.py index 06959f29..9e053a47 100644 --- a/taskflow/engines/action_engine/runtime.py +++ b/taskflow/engines/action_engine/runtime.py @@ -36,8 +36,8 @@ class Runtime(object): action engine to run to completion. """ - def __init__(self, compilation, storage, task_notifier, task_executor): - self._task_notifier = task_notifier + def __init__(self, compilation, storage, atom_notifier, task_executor): + self._atom_notifier = atom_notifier self._task_executor = task_executor self._storage = storage self._compilation = compilation @@ -68,7 +68,7 @@ class Runtime(object): @misc.cachedproperty def retry_action(self): - return ra.RetryAction(self._storage, self._task_notifier, + return ra.RetryAction(self._storage, self._atom_notifier, lambda atom: sc.ScopeWalker(self.compilation, atom, names_only=True)) @@ -76,7 +76,7 @@ class Runtime(object): @misc.cachedproperty def task_action(self): return ta.TaskAction(self._storage, self._task_executor, - self._task_notifier, + self._atom_notifier, lambda atom: sc.ScopeWalker(self.compilation, atom, names_only=True)) diff --git a/taskflow/engines/base.py b/taskflow/engines/base.py index 2fab9d0f..2c409726 100644 --- a/taskflow/engines/base.py +++ b/taskflow/engines/base.py @@ -20,6 +20,7 @@ import abc import six from taskflow.types import notifier +from taskflow.utils import deprecation from taskflow.utils import misc @@ -31,6 +32,10 @@ class EngineBase(object): occur related to the flow the engine contains. :ivar task_notifier: A notification object that will dispatch events that occur related to the tasks the engine contains. + occur related to the tasks the engine + contains (deprecated). + :ivar atom_notifier: A notification object that will dispatch events that + occur related to the atoms the engine contains. """ def __init__(self, flow, flow_detail, backend, options): @@ -41,8 +46,25 @@ class EngineBase(object): self._options = {} else: self._options = dict(options) - self.notifier = notifier.Notifier() - self.task_notifier = notifier.Notifier() + self._notifier = notifier.Notifier() + self._atom_notifier = notifier.Notifier() + + @property + def notifier(self): + """The flow notifier.""" + return self._notifier + + @property + @deprecation.moved_property('atom_notifier', version="0.6", + removal_version="?") + def task_notifier(self): + """The task notifier.""" + return self._atom_notifier + + @property + def atom_notifier(self): + """The atom notifier.""" + return self._atom_notifier @property def options(self): diff --git a/taskflow/utils/deprecation.py b/taskflow/utils/deprecation.py index b9a1c724..b83cce58 100644 --- a/taskflow/utils/deprecation.py +++ b/taskflow/utils/deprecation.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +import functools import warnings import wrapt @@ -47,6 +48,48 @@ class MovedClassProxy(wrapt.ObjectProxy): return issubclass(instance, self.__wrapped__) +def _generate_moved_message(kind, old_name, new_name, + message=None, version=None, removal_version=None): + message_components = [ + "%s '%s' has moved to '%s'" % (kind, old_name, new_name), + ] + if version: + message_components.append(" in version '%s'" % version) + if removal_version: + if removal_version == "?": + message_components.append(" and will be removed in a future" + " version") + else: + message_components.append(" and will be removed in version" + " '%s'" % removal_version) + if message: + message_components.append(": %s" % message) + return ''.join(message_components) + + +def _moved_decorator(kind, new_name, message=None, + version=None, removal_version=None): + """Decorates a method/function/other that was moved to another location.""" + + @wrapt.decorator + def decorator(wrapped, instance, args, kwargs): + try: + old_name = wrapped.__qualname__ + except AttributeError: + old_name = wrapped.__name__ + out_message = _generate_moved_message(kind, old_name, new_name, + message=message, version=version, + removal_version=removal_version) + deprecation(out_message, 3) + return wrapped(*args, **kwargs) + + return decorator + + +"""Decorates a property that was moved to another location.""" +moved_property = functools.partial(_moved_decorator, 'Property') + + def moved_class(new_class, old_class_name, old_module_name, message=None, version=None, removal_version=None): """Deprecates a class that was moved to another location. @@ -56,18 +99,7 @@ def moved_class(new_class, old_class_name, old_module_name, message=None, """ old_name = ".".join((old_module_name, old_class_name)) new_name = reflection.get_class_name(new_class) - message_components = [ - "Class '%s' has moved to '%s'" % (old_name, new_name), - ] - if version: - message_components.append(" in version '%s'" % version) - if removal_version: - if removal_version == "?": - message_components.append(" and will be removed in a future" - " version") - else: - message_components.append(" and will be removed in version '%s'" - % removal_version) - if message: - message_components.append(": %s" % message) - return MovedClassProxy(new_class, "".join(message_components), 3) + out_message = _generate_moved_message('Class', old_name, new_name, + message=message, version=version, + removal_version=removal_version) + return MovedClassProxy(new_class, out_message, 3) From 613af6131357265157517d189e276070b1bdf8af Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 6 Nov 2014 02:23:25 -0800 Subject: [PATCH 096/240] Ensure failure types contain only immutable items Always make sure 'exc_type_names' is a tuple, when it comes from either the 'exc_info' tuple or from **kwargs to enforce the failure type being immutable (which it always should be). Change-Id: Icbdd9bc67d26b25f510914d0b19df9caef400c6d --- taskflow/types/failure.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/taskflow/types/failure.py b/taskflow/types/failure.py index 449bf9b3..c905277c 100644 --- a/taskflow/types/failure.py +++ b/taskflow/types/failure.py @@ -114,8 +114,15 @@ class Failure(object): if not kwargs: if exc_info is None: exc_info = sys.exc_info() + else: + # This should always be the (type, value, traceback) tuple, + # either from a prior sys.exc_info() call or from some other + # creation... + if len(exc_info) != 3: + raise ValueError("Provided 'exc_info' must contain three" + " elements") self._exc_info = exc_info - self._exc_type_names = list( + self._exc_type_names = tuple( reflection.get_all_class_names(exc_info[0], up_to=Exception)) if not self._exc_type_names: raise TypeError('Invalid exception type: %r' % exc_info[0]) @@ -125,7 +132,7 @@ class Failure(object): else: self._exc_info = exc_info # may be None self._exception_str = kwargs.pop('exception_str') - self._exc_type_names = kwargs.pop('exc_type_names', []) + self._exc_type_names = tuple(kwargs.pop('exc_type_names', [])) self._traceback_str = kwargs.pop('traceback_str', None) if kwargs: raise TypeError( From c543dc20669aa4de645251d0418fd0e6e42b9557 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 9 Oct 2014 14:28:59 -0700 Subject: [PATCH 097/240] When creating daemon threads use the bundled threading_utils Instead of creating daemon threads using the threads module directly use our small utility file to create the daemon thread on our behalf and set the appropriate attributes to ensure it's a daemon thread. This change replaces the existing locations where we were doing this manually and uses the threading_utils helper function uniformly instead. Change-Id: I535cee8a63407f753cf812df53c4f5bc83e0c9ae --- .../examples/jobboard_produce_consume_colors.py | 7 +++---- taskflow/examples/wbe_mandelbrot.py | 5 ++--- taskflow/examples/wbe_simple_linear.py | 5 ++--- taskflow/tests/unit/conductor/test_conductor.py | 13 +++---------- taskflow/tests/unit/jobs/base.py | 7 ++----- taskflow/tests/unit/test_engines.py | 2 +- taskflow/tests/unit/test_utils.py | 5 ++--- taskflow/tests/unit/test_utils_lock_utils.py | 13 +++++-------- .../tests/unit/worker_based/test_message_pump.py | 11 +++-------- taskflow/tests/unit/worker_based/test_pipeline.py | 6 ++---- taskflow/tests/unit/worker_based/test_proxy.py | 5 ++--- 11 files changed, 27 insertions(+), 52 deletions(-) diff --git a/taskflow/examples/jobboard_produce_consume_colors.py b/taskflow/examples/jobboard_produce_consume_colors.py index aa80828f..80c2acba 100644 --- a/taskflow/examples/jobboard_produce_consume_colors.py +++ b/taskflow/examples/jobboard_produce_consume_colors.py @@ -35,6 +35,7 @@ from zake import fake_client from taskflow import exceptions as excp from taskflow.jobs import backends +from taskflow.utils import threading_utils # In this example we show how a jobboard can be used to post work for other # entities to work on. This example creates a set of jobs using one producer @@ -152,14 +153,12 @@ def main(): with contextlib.closing(fake_client.FakeClient()) as c: created = [] for i in compat_range(0, PRODUCERS): - p = threading.Thread(target=producer, args=(i + 1, c)) - p.daemon = True + p = threading_utils.daemon_thread(producer, i + 1, c) created.append(p) p.start() consumed = collections.deque() for i in compat_range(0, WORKERS): - w = threading.Thread(target=worker, args=(i + 1, c, consumed)) - w.daemon = True + w = threading_utils.daemon_thread(worker, i + 1, c, consumed) created.append(w) w.start() while created: diff --git a/taskflow/examples/wbe_mandelbrot.py b/taskflow/examples/wbe_mandelbrot.py index cf46c240..f21465e3 100644 --- a/taskflow/examples/wbe_mandelbrot.py +++ b/taskflow/examples/wbe_mandelbrot.py @@ -18,7 +18,6 @@ import logging import math import os import sys -import threading top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, @@ -31,6 +30,7 @@ from taskflow import engines from taskflow.engines.worker_based import worker from taskflow.patterns import unordered_flow as uf from taskflow import task +from taskflow.utils import threading_utils # INTRO: This example walks through a workflow that will in parallel compute # a mandelbrot result set (using X 'remote' workers) and then combine their @@ -229,8 +229,7 @@ def create_fractal(): worker_conf['topic'] = 'calculator_%s' % (i + 1) worker_topics.append(worker_conf['topic']) w = worker.Worker(**worker_conf) - runner = threading.Thread(target=w.run) - runner.daemon = True + runner = threading_utils.daemon_thread(w.run) runner.start() w.wait() workers.append((runner, w.stop)) diff --git a/taskflow/examples/wbe_simple_linear.py b/taskflow/examples/wbe_simple_linear.py index a15b48fa..bcaa8612 100644 --- a/taskflow/examples/wbe_simple_linear.py +++ b/taskflow/examples/wbe_simple_linear.py @@ -19,7 +19,6 @@ import logging import os import sys import tempfile -import threading top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, @@ -30,6 +29,7 @@ from taskflow import engines from taskflow.engines.worker_based import worker from taskflow.patterns import linear_flow as lf from taskflow.tests import utils +from taskflow.utils import threading_utils import example_utils # noqa @@ -123,8 +123,7 @@ if __name__ == "__main__": worker_conf['topic'] = 'worker-%s' % (i + 1) worker_topics.append(worker_conf['topic']) w = worker.Worker(**worker_conf) - runner = threading.Thread(target=w.run) - runner.daemon = True + runner = threading_utils.daemon_thread(w.run) runner.start() w.wait() workers.append((runner, w.stop)) diff --git a/taskflow/tests/unit/conductor/test_conductor.py b/taskflow/tests/unit/conductor/test_conductor.py index cf19fa84..0fb677ea 100644 --- a/taskflow/tests/unit/conductor/test_conductor.py +++ b/taskflow/tests/unit/conductor/test_conductor.py @@ -16,7 +16,6 @@ import collections import contextlib -import threading from zake import fake_client @@ -51,12 +50,6 @@ def test_factory(blowup): return f -def make_thread(conductor): - t = threading.Thread(target=conductor.run) - t.daemon = True - return t - - class SingleThreadedConductorTest(test_utils.EngineTestBase, test.TestCase): ComponentBundle = collections.namedtuple('ComponentBundle', ['board', 'client', @@ -85,7 +78,7 @@ class SingleThreadedConductorTest(test_utils.EngineTestBase, test.TestCase): components = self.make_components() components.conductor.connect() with close_many(components.conductor, components.client): - t = make_thread(components.conductor) + t = threading_utils.daemon_thread(components.conductor.run) t.start() self.assertTrue( components.conductor.stop(test_utils.WAIT_TIMEOUT)) @@ -102,7 +95,7 @@ class SingleThreadedConductorTest(test_utils.EngineTestBase, test.TestCase): components.board.notifier.register(jobboard.REMOVAL, on_consume) with close_many(components.conductor, components.client): - t = make_thread(components.conductor) + t = threading_utils.daemon_thread(components.conductor.run) t.start() lb, fd = pu.temporary_flow_detail(components.persistence) engines.save_factory_details(fd, test_factory, @@ -131,7 +124,7 @@ class SingleThreadedConductorTest(test_utils.EngineTestBase, test.TestCase): components.board.notifier.register(jobboard.REMOVAL, on_consume) with close_many(components.conductor, components.client): - t = make_thread(components.conductor) + t = threading_utils.daemon_thread(components.conductor.run) t.start() lb, fd = pu.temporary_flow_detail(components.persistence) engines.save_factory_details(fd, test_factory, diff --git a/taskflow/tests/unit/jobs/base.py b/taskflow/tests/unit/jobs/base.py index e0a20a04..24da7d3a 100644 --- a/taskflow/tests/unit/jobs/base.py +++ b/taskflow/tests/unit/jobs/base.py @@ -15,7 +15,6 @@ # under the License. import contextlib -import threading import time from kazoo.recipe import watchers @@ -143,11 +142,9 @@ class BoardTestMixin(object): jobs.extend(it) with connect_close(self.board): - t1 = threading.Thread(target=poster) - t1.daemon = True + t1 = threading_utils.daemon_thread(poster) t1.start() - t2 = threading.Thread(target=waiter) - t2.daemon = True + t2 = threading_utils.daemon_thread(waiter) t2.start() for t in (t1, t2): t.join() diff --git a/taskflow/tests/unit/test_engines.py b/taskflow/tests/unit/test_engines.py index 43153f36..1ff81353 100644 --- a/taskflow/tests/unit/test_engines.py +++ b/taskflow/tests/unit/test_engines.py @@ -644,7 +644,7 @@ class WorkerBasedEngineTest(EngineTaskTest, 'topics': tuple([worker_conf['topic']]), }) self.worker = wkr.Worker(**worker_conf) - self.worker_thread = tu.daemon_thread(target=self.worker.run) + self.worker_thread = tu.daemon_thread(self.worker.run) self.worker_thread.start() # Make sure the worker is started before we can continue... diff --git a/taskflow/tests/unit/test_utils.py b/taskflow/tests/unit/test_utils.py index 38417810..66f08c09 100644 --- a/taskflow/tests/unit/test_utils.py +++ b/taskflow/tests/unit/test_utils.py @@ -17,7 +17,6 @@ import collections import inspect import random -import threading import time import six @@ -29,6 +28,7 @@ from taskflow.types import failure from taskflow.utils import lock_utils from taskflow.utils import misc from taskflow.utils import reflection +from taskflow.utils import threading_utils def mere_function(a, b): @@ -373,8 +373,7 @@ class CachedPropertyTest(test.TestCase): threads = [] try: for _i in range(0, 20): - t = threading.Thread(target=lambda: a.b) - t.daemon = True + t = threading_utils.daemon_thread(lambda: a.b) threads.append(t) for t in threads: t.start() diff --git a/taskflow/tests/unit/test_utils_lock_utils.py b/taskflow/tests/unit/test_utils_lock_utils.py index 37bc1711..06bef1ee 100644 --- a/taskflow/tests/unit/test_utils_lock_utils.py +++ b/taskflow/tests/unit/test_utils_lock_utils.py @@ -195,8 +195,7 @@ class MultilockTest(test.TestCase): threads = [] for _i in range(0, 20): - t = threading.Thread(target=run) - t.daemon = True + t = threading_utils.daemon_thread(run) threads.append(t) t.start() while threads: @@ -234,9 +233,8 @@ class MultilockTest(test.TestCase): target = run_fail else: target = run - t = threading.Thread(target=target) + t = threading_utils.daemon_thread(target) threads.append(t) - t.daemon = True t.start() while threads: t = threads.pop() @@ -287,9 +285,8 @@ class MultilockTest(test.TestCase): threads = [] for i in range(0, 20): - t = threading.Thread(target=run) + t = threading_utils.daemon_thread(run) threads.append(t) - t.daemon = True t.start() while threads: t = threads.pop() @@ -369,11 +366,11 @@ class ReadWriteLockTest(test.TestCase): with lock.write_lock(): activated.append(lock.owner) - reader = threading.Thread(target=double_reader) + reader = threading_utils.daemon_thread(double_reader) reader.start() self.assertTrue(active.wait(test_utils.WAIT_TIMEOUT)) - writer = threading.Thread(target=happy_writer) + writer = threading_utils.daemon_thread(happy_writer) writer.start() reader.join() diff --git a/taskflow/tests/unit/worker_based/test_message_pump.py b/taskflow/tests/unit/worker_based/test_message_pump.py index 1fc946ed..8b48b435 100644 --- a/taskflow/tests/unit/worker_based/test_message_pump.py +++ b/taskflow/tests/unit/worker_based/test_message_pump.py @@ -14,8 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -import threading - from taskflow.engines.worker_based import protocol as pr from taskflow.engines.worker_based import proxy from taskflow.openstack.common import uuidutils @@ -43,8 +41,7 @@ class TestMessagePump(test.TestCase): 'polling_interval': POLLING_INTERVAL, }) - t = threading.Thread(target=p.start) - t.daemon = True + t = threading_utils.daemon_thread(p.start) t.start() p.wait() p.publish(pr.Notify(), TEST_TOPIC) @@ -69,8 +66,7 @@ class TestMessagePump(test.TestCase): 'polling_interval': POLLING_INTERVAL, }) - t = threading.Thread(target=p.start) - t.daemon = True + t = threading_utils.daemon_thread(p.start) t.start() p.wait() resp = pr.Response(pr.RUNNING) @@ -109,8 +105,7 @@ class TestMessagePump(test.TestCase): 'polling_interval': POLLING_INTERVAL, }) - t = threading.Thread(target=p.start) - t.daemon = True + t = threading_utils.daemon_thread(p.start) t.start() p.wait() diff --git a/taskflow/tests/unit/worker_based/test_pipeline.py b/taskflow/tests/unit/worker_based/test_pipeline.py index b86cedd0..ed3e2662 100644 --- a/taskflow/tests/unit/worker_based/test_pipeline.py +++ b/taskflow/tests/unit/worker_based/test_pipeline.py @@ -14,8 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -import threading - from concurrent import futures from taskflow.engines.worker_based import endpoint @@ -25,6 +23,7 @@ from taskflow.openstack.common import uuidutils from taskflow import test from taskflow.tests import utils as test_utils from taskflow.types import failure +from taskflow.utils import threading_utils TEST_EXCHANGE, TEST_TOPIC = ('test-exchange', 'test-topic') @@ -44,8 +43,7 @@ class TestPipeline(test.TestCase): transport_options={ 'polling_interval': POLLING_INTERVAL, }) - server_thread = threading.Thread(target=server.start) - server_thread.daemon = True + server_thread = threading_utils.daemon_thread(server.start) return (server, server_thread) def _fetch_executor(self): diff --git a/taskflow/tests/unit/worker_based/test_proxy.py b/taskflow/tests/unit/worker_based/test_proxy.py index de5f3abc..4217a726 100644 --- a/taskflow/tests/unit/worker_based/test_proxy.py +++ b/taskflow/tests/unit/worker_based/test_proxy.py @@ -15,12 +15,12 @@ # under the License. import socket -import threading from six.moves import mock from taskflow.engines.worker_based import proxy from taskflow import test +from taskflow.utils import threading_utils class TestProxy(test.MockTestCase): @@ -210,8 +210,7 @@ class TestProxy(test.MockTestCase): self.assertFalse(pr.is_running) # start proxy in separate thread - t = threading.Thread(target=pr.start) - t.daemon = True + t = threading_utils.daemon_thread(pr.start) t.start() # make sure proxy is started From 7d199e0ddb17a8ecdd5269f2134e0b40c804ba1b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 26 Oct 2014 15:21:30 -0700 Subject: [PATCH 098/240] Format failures via a static method To allow for customized formatting of failures create and use a tiny helper function that can be used to alter how failures are formatted to better suite the users needs. This allows for users to dervive from the top level class and adjust there formatting for how they see fit (as long as they return the same two values that the methods api defines). Change-Id: Ib91bd28ff4f24d831bb97715282ee8f3b6050c4c --- taskflow/listeners/logging.py | 65 +++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/taskflow/listeners/logging.py b/taskflow/listeners/logging.py index 3629bb2c..6264333b 100644 --- a/taskflow/listeners/logging.py +++ b/taskflow/listeners/logging.py @@ -110,49 +110,71 @@ class DynamicLoggingListener(base.ListenerBase): flow_listen_for=flow_listen_for) self._failure_level = failure_level self._level = level + self._task_log_levels = { + states.FAILURE: self._failure_level, + states.REVERTED: self._failure_level, + states.RETRYING: self._failure_level, + } + self._flow_log_levels = { + states.FAILURE: self._failure_level, + states.REVERTED: self._failure_level, + } if not log: self._logger = LOG else: self._logger = log + @staticmethod + def _format_failure(fail): + """Returns a (exc_info, exc_details) tuple about the failure. + + The ``exc_info`` tuple should be a standard three element + (exctype, value, traceback) tuple that will be used for further + logging. If a non-empty string is returned for ``exc_details`` it + should contain any string info about the failure (with any specific + details the ``exc_info`` may not have/contain). If the ``exc_info`` + tuple is returned as ``None`` then it will cause the logging + system to avoid outputting any traceback information (read + the python documentation on the logger interaction with ``exc_info`` + to learn more). + """ + if fail.exc_info: + exc_info = fail.exc_info + exc_details = '' + else: + # When a remote failure occurs (or somehow the failure + # object lost its traceback), we will not have a valid + # exc_info that can be used but we *should* have a string + # version that we can use instead... + exc_info = None + exc_details = "\n%s" % fail.pformat(traceback=True) + return (exc_info, exc_details) + def _flow_receiver(self, state, details): - # Gets called on flow state changes. - level = self._level - if state in (states.FAILURE, states.REVERTED): - level = self._failure_level + """Gets called on flow state changes.""" + level = self._flow_log_levels.get(state, self._level) self._logger.log(level, "Flow '%s' (%s) transitioned into state '%s'" " from state '%s'", details['flow_name'], details['flow_uuid'], state, details.get('old_state')) def _task_receiver(self, state, details): - # Gets called on task state changes. + """Gets called on task state changes.""" if 'result' in details and state in base.FINISH_STATES: # If the task failed, it's useful to show the exception traceback # and any other available exception information. result = details.get('result') if isinstance(result, failure.Failure): - if result.exc_info: - exc_info = result.exc_info - manual_tb = '' - else: - # When a remote failure occurs (or somehow the failure - # object lost its traceback), we will not have a valid - # exc_info that can be used but we *should* have a string - # version that we can use instead... - exc_info = None - manual_tb = "\n%s" % result.pformat(traceback=True) + exc_info, exc_details = self._format_failure(result) self._logger.log(self._failure_level, "Task '%s' (%s) transitioned into state" " '%s'%s", details['task_name'], - details['task_uuid'], state, manual_tb, + details['task_uuid'], state, exc_details, exc_info=exc_info) else: # Otherwise, depending on the enabled logging level/state we # will show or hide results that the task may have produced # during execution. - level = self._level - if state == states.FAILURE: - level = self._failure_level + level = self._task_log_levels.get(state, self._level) if (_isEnabledFor(self._logger, self._level) or state == states.FAILURE): self._logger.log(level, "Task '%s' (%s) transitioned into" @@ -165,9 +187,8 @@ class DynamicLoggingListener(base.ListenerBase): " state '%s'", details['task_name'], details['task_uuid'], state) else: - level = self._level - if state in (states.REVERTING, states.RETRYING): - level = self._failure_level + # Just a intermediary state, carry on! + level = self._task_log_levels.get(state, self._level) self._logger.log(level, "Task '%s' (%s) transitioned into state" " '%s'", details['task_name'], details['task_uuid'], state) From 2a7ca479676e1293fefb707788e17b37b17770f4 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 23 Sep 2014 16:55:05 -0700 Subject: [PATCH 099/240] Add a history retry object, makes retry histories easier to use When a retry object is asked to make a decision about the atoms it controls it is currently provided a complex object that contains what has failed and why and what was provided to try to resolve this failure as raw types. To make it easier to interact with that history provide and use a more easier to interact with helper object that provides useful functionality built ontop of the raw types. Part of blueprint intuitive-retries Change-Id: I93f86552f5a0c26b269319e4de6d9b8fb3b3b219 --- doc/source/arguments_and_results.rst | 30 ++++++------- taskflow/retry.py | 67 +++++++++++++++++++++++++++- taskflow/storage.py | 35 ++++++++++----- taskflow/tests/unit/test_retries.py | 12 ++--- taskflow/tests/unit/test_storage.py | 22 +++++---- 5 files changed, 126 insertions(+), 40 deletions(-) diff --git a/doc/source/arguments_and_results.rst b/doc/source/arguments_and_results.rst index 3082e2fc..cb2c8761 100644 --- a/doc/source/arguments_and_results.rst +++ b/doc/source/arguments_and_results.rst @@ -385,11 +385,13 @@ representation of the ``do_something`` result. Retry arguments =============== -A |Retry| controller works with arguments in the same way as a |Task|. But -it has an additional parameter ``'history'`` that is a list of tuples. Each -tuple contains a result of the previous retry run and a table where the key -is a failed task and the value is a -:py:class:`~taskflow.types.failure.Failure` object. +A |Retry| controller works with arguments in the same way as a |Task|. But it +has an additional parameter ``'history'`` that is itself a +:py:class:`~taskflow.retry.History` object that contains what failed over all +the engines attempts (aka the outcomes). The history object can be +viewed as a tuple that contains a result of the previous retrys run and a +table/dict where each key is a failed atoms name and each value is +a :py:class:`~taskflow.types.failure.Failure` object. Consider the following implementation:: @@ -398,19 +400,19 @@ Consider the following implementation:: default_provides = 'value' def on_failure(self, history, *args, **kwargs): - print history + print(list(history)) return RETRY def execute(self, history, *args, **kwargs): - print history + print(list(history)) return 5 def revert(self, history, *args, **kwargs): - print history + print(list(history)) Imagine the above retry had returned a value ``'5'`` and then some task ``'A'`` -failed with some exception. In this case the above retrys ``on_failure`` -method will receive the following history:: +failed with some exception. In this case ``on_failure`` method will receive +the following history (printed as a list):: [('5', {'A': failure.Failure()})] @@ -419,12 +421,10 @@ At this point (since the implementation returned ``RETRY``) the history and it can then return a value that subseqent tasks can use to alter there behavior. -If instead the |retry.execute| method raises an exception, the |retry.revert| -method of the implementation will be called and +If instead the |retry.execute| method itself raises an exception, +the |retry.revert| method of the implementation will be called and a :py:class:`~taskflow.types.failure.Failure` object will be present in the -history instead of the typical result:: - - [('5', {'A': failure.Failure()}), (failure.Failure(), {})] +history object instead of the typical result. .. note:: diff --git a/taskflow/retry.py b/taskflow/retry.py index 6897e3b7..4dab636f 100644 --- a/taskflow/retry.py +++ b/taskflow/retry.py @@ -41,6 +41,70 @@ EXECUTE_REVERT_HISTORY = 'history' REVERT_FLOW_FAILURES = 'flow_failures' +class History(object): + """Helper that simplifies interactions with retry historical contents.""" + + def __init__(self, contents, failure=None): + self._contents = contents + self._failure = failure + + @property + def failure(self): + """Returns the retries own failure or none if not existent.""" + return self._failure + + def outcomes_iter(self, index=None): + """Iterates over the contained failure outcomes. + + If the index is not provided, then all outcomes are iterated over. + + NOTE(harlowja): if the retry itself failed, this will **not** include + those types of failures. Use the :py:attr:`.failure` attribute to + access that instead (if it exists, aka, non-none). + """ + if index is None: + contents = self._contents + else: + contents = [ + self._contents[index], + ] + for (provided, outcomes) in contents: + for (owner, outcome) in six.iteritems(outcomes): + yield (owner, outcome) + + def __len__(self): + return len(self._contents) + + def provided_iter(self): + """Iterates over all the values the retry has attempted (in order).""" + for (provided, outcomes) in self._contents: + yield provided + + def __getitem__(self, index): + return self._contents[index] + + def caused_by(self, exception_cls, index=None, include_retry=False): + """Checks if the exception class provided caused the failures. + + If the index is not provided, then all outcomes are iterated over. + + NOTE(harlowja): only if ``include_retry`` is provided as true (defaults + to false) will the potential retries own failure be + checked against as well. + """ + for (name, failure) in self.outcomes_iter(index=index): + if failure.check(exception_cls): + return True + if include_retry and self._failure is not None: + if self._failure.check(exception_cls): + return True + return False + + def __iter__(self): + """Iterates over the raw contents of this history object.""" + return iter(self._contents) + + @six.add_metaclass(abc.ABCMeta) class Retry(atom.Atom): """A class that can decide how to resolve execution failures. @@ -177,8 +241,7 @@ class ForEachBase(Retry): # Fetches the next resolution result to try, removes overlapping # entries with what has already been tried and then returns the first # resolution strategy remaining. - items = (item for item, _failures in history) - remaining = misc.sequence_minus(values, items) + remaining = misc.sequence_minus(values, history.provided_iter()) if not remaining: raise exc.NotFound("No elements left in collection of iterable " "retry controller %s" % self.name) diff --git a/taskflow/storage.py b/taskflow/storage.py index c667509b..30b12951 100644 --- a/taskflow/storage.py +++ b/taskflow/storage.py @@ -745,20 +745,35 @@ class Storage(object): state = states.PENDING return state + def _translate_into_history(self, ad): + failure = None + if ad.failure is not None: + # NOTE(harlowja): Try to use our local cache to get a more + # complete failure object that has a traceback (instead of the + # one that is saved which will *typically* not have one)... + cached = self._failures.get(ad.name) + if ad.failure.matches(cached): + failure = cached + else: + failure = ad.failure + return retry.History(ad.results, failure=failure) + def get_retry_history(self, retry_name): - """Fetch retry results history.""" + """Fetch a single retrys history.""" with self._lock.read_lock(): ad = self._atomdetail_by_name(retry_name, expected_type=logbook.RetryDetail) - if ad.failure is not None: - cached = self._failures.get(retry_name) - history = list(ad.results) - if ad.failure.matches(cached): - history.append((cached, {})) - else: - history.append((ad.failure, {})) - return history - return ad.results + return self._translate_into_history(ad) + + def get_retry_histories(self): + """Fetch all retrys histories.""" + histories = [] + with self._lock.read_lock(): + for ad in self._flowdetail: + if isinstance(ad, logbook.RetryDetail): + histories.append((ad.name, + self._translate_into_history(ad))) + return histories class MultiThreadedStorage(Storage): diff --git a/taskflow/tests/unit/test_retries.py b/taskflow/tests/unit/test_retries.py index 54400435..106bfaed 100644 --- a/taskflow/tests/unit/test_retries.py +++ b/taskflow/tests/unit/test_retries.py @@ -631,11 +631,13 @@ class RetryTest(utils.EngineTestBase): r = FailingRetry() flow = lf.Flow('testflow', r) - self.assertRaisesRegexp(ValueError, '^OMG', - self._make_engine(flow).run) - self.assertEqual(len(r.history), 1) - self.assertEqual(r.history[0][1], {}) - self.assertEqual(isinstance(r.history[0][0], failure.Failure), True) + engine = self._make_engine(flow) + self.assertRaisesRegexp(ValueError, '^OMG', engine.run) + self.assertEqual(1, len(engine.storage.get_retry_histories())) + self.assertEqual(len(r.history), 0) + self.assertEqual([], list(r.history.outcomes_iter())) + self.assertIsNotNone(r.history.failure) + self.assertTrue(r.history.caused_by(ValueError, include_retry=True)) def test_retry_revert_fails(self): diff --git a/taskflow/tests/unit/test_storage.py b/taskflow/tests/unit/test_storage.py index deb4db4f..f774993c 100644 --- a/taskflow/tests/unit/test_storage.py +++ b/taskflow/tests/unit/test_storage.py @@ -448,7 +448,7 @@ class StorageTestMixin(object): s = self._get_storage() s.ensure_atom(test_utils.NoopRetry('my retry')) history = s.get_retry_history('my retry') - self.assertEqual(history, []) + self.assertEqual([], list(history)) def test_ensure_retry_and_task_with_same_name(self): s = self._get_storage() @@ -463,7 +463,8 @@ class StorageTestMixin(object): s.save('my retry', 'a') s.save('my retry', 'b') history = s.get_retry_history('my retry') - self.assertEqual(history, [('a', {}), ('b', {})]) + self.assertEqual([('a', {}), ('b', {})], list(history)) + self.assertEqual(['a', 'b'], list(history.provided_iter())) def test_save_retry_results_with_mapping(self): s = self._get_storage() @@ -471,9 +472,10 @@ class StorageTestMixin(object): s.save('my retry', 'a') s.save('my retry', 'b') history = s.get_retry_history('my retry') - self.assertEqual(history, [('a', {}), ('b', {})]) - self.assertEqual(s.fetch_all(), {'x': 'b'}) - self.assertEqual(s.fetch('x'), 'b') + self.assertEqual([('a', {}), ('b', {})], list(history)) + self.assertEqual(['a', 'b'], list(history.provided_iter())) + self.assertEqual({'x': 'b'}, s.fetch_all()) + self.assertEqual('b', s.fetch('x')) def test_cleanup_retry_history(self): s = self._get_storage() @@ -482,7 +484,8 @@ class StorageTestMixin(object): s.save('my retry', 'b') s.cleanup_retry_history('my retry', states.REVERTED) history = s.get_retry_history('my retry') - self.assertEqual(history, []) + self.assertEqual(list(history), []) + self.assertEqual(0, len(history)) self.assertEqual(s.fetch_all(), {}) def test_cached_retry_failure(self): @@ -492,8 +495,11 @@ class StorageTestMixin(object): s.save('my retry', 'a') s.save('my retry', a_failure, states.FAILURE) history = s.get_retry_history('my retry') - self.assertEqual(history, [('a', {}), (a_failure, {})]) - self.assertIs(s.has_failures(), True) + self.assertEqual([('a', {})], list(history)) + self.assertTrue(history.caused_by(RuntimeError, include_retry=True)) + self.assertIsNotNone(history.failure) + self.assertEqual(1, len(history)) + self.assertTrue(s.has_failures()) self.assertEqual(s.get_failures(), {'my retry': a_failure}) def test_logbook_get_unknown_atom_type(self): From a77d192f9c06a19df763d03e58d0a660b54e6a91 Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Thu, 20 Nov 2014 14:12:51 +0000 Subject: [PATCH 100/240] Updated from global requirements Change-Id: Ide0e95f47a88015801e445c1cd659f30c51e0c0b --- requirements-py2.txt | 2 +- requirements-py3.txt | 2 +- test-requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-py2.txt b/requirements-py2.txt index a1de6663..d4469f8c 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -14,7 +14,7 @@ ordereddict six>=1.7.0 # For proxying objects and making correct decorators -wrapt>=1.7.0 +wrapt>=1.7.0 # BSD License # Very nice graph library networkx>=1.8 diff --git a/requirements-py3.txt b/requirements-py3.txt index 549a5dfa..174eb59c 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -11,7 +11,7 @@ pbr>=0.6,!=0.7,<1.0 six>=1.7.0 # For proxying objects and making correct decorators -wrapt>=1.7.0 +wrapt>=1.7.0 # BSD License # Very nice graph library networkx>=1.8 diff --git a/test-requirements.txt b/test-requirements.txt index 3a669cb4..36857441 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,7 +5,7 @@ hacking>=0.9.2,<0.10 oslotest>=1.2.0 # Apache-2.0 mock>=1.0 -testtools>=0.9.36,!=1.2.0,!=1.4.0 +testtools>=0.9.36,!=1.2.0 # Used for testing the WBE engine. kombu>=2.5.0 From 2f7e58235bf55aa3072b6b736b2dacc7ff3a7e1b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 20 Nov 2014 15:26:00 -0800 Subject: [PATCH 101/240] Revert wrapt usage until further notice Currently its usage is failing on stable/icehouse which is being fixed, but that branch is also currently broken due to a tempest failure; so until this situation is resolved take out the usage of wrapt. Part of fixes for bug 1394647 Change-Id: Ibfe21944b6e6882f19f7cf4359e8356a64200278 --- requirements-py2.txt | 3 - requirements-py3.txt | 3 - taskflow/engines/action_engine/engine.py | 2 +- taskflow/engines/action_engine/runtime.py | 8 +- taskflow/engines/base.py | 26 +---- taskflow/tests/unit/test_failure.py | 12 -- taskflow/tests/unit/test_notifier.py | 9 -- taskflow/utils/deprecation.py | 127 ++++++++++++---------- 8 files changed, 77 insertions(+), 113 deletions(-) diff --git a/requirements-py2.txt b/requirements-py2.txt index d4469f8c..0827e7ed 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -13,9 +13,6 @@ ordereddict # Python 2->3 compatibility library. six>=1.7.0 -# For proxying objects and making correct decorators -wrapt>=1.7.0 # BSD License - # Very nice graph library networkx>=1.8 diff --git a/requirements-py3.txt b/requirements-py3.txt index 174eb59c..5f265dcc 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -10,9 +10,6 @@ pbr>=0.6,!=0.7,<1.0 # Python 2->3 compatibility library. six>=1.7.0 -# For proxying objects and making correct decorators -wrapt>=1.7.0 # BSD License - # Very nice graph library networkx>=1.8 diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index 5a0fc9fb..730d2895 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -217,7 +217,7 @@ class ActionEngine(base.EngineBase): self._compilation = self._compiler.compile() self._runtime = runtime.Runtime(self._compilation, self.storage, - self.atom_notifier, + self.task_notifier, self._task_executor) self._compiled = True diff --git a/taskflow/engines/action_engine/runtime.py b/taskflow/engines/action_engine/runtime.py index 9e053a47..06959f29 100644 --- a/taskflow/engines/action_engine/runtime.py +++ b/taskflow/engines/action_engine/runtime.py @@ -36,8 +36,8 @@ class Runtime(object): action engine to run to completion. """ - def __init__(self, compilation, storage, atom_notifier, task_executor): - self._atom_notifier = atom_notifier + def __init__(self, compilation, storage, task_notifier, task_executor): + self._task_notifier = task_notifier self._task_executor = task_executor self._storage = storage self._compilation = compilation @@ -68,7 +68,7 @@ class Runtime(object): @misc.cachedproperty def retry_action(self): - return ra.RetryAction(self._storage, self._atom_notifier, + return ra.RetryAction(self._storage, self._task_notifier, lambda atom: sc.ScopeWalker(self.compilation, atom, names_only=True)) @@ -76,7 +76,7 @@ class Runtime(object): @misc.cachedproperty def task_action(self): return ta.TaskAction(self._storage, self._task_executor, - self._atom_notifier, + self._task_notifier, lambda atom: sc.ScopeWalker(self.compilation, atom, names_only=True)) diff --git a/taskflow/engines/base.py b/taskflow/engines/base.py index 2c409726..2fab9d0f 100644 --- a/taskflow/engines/base.py +++ b/taskflow/engines/base.py @@ -20,7 +20,6 @@ import abc import six from taskflow.types import notifier -from taskflow.utils import deprecation from taskflow.utils import misc @@ -32,10 +31,6 @@ class EngineBase(object): occur related to the flow the engine contains. :ivar task_notifier: A notification object that will dispatch events that occur related to the tasks the engine contains. - occur related to the tasks the engine - contains (deprecated). - :ivar atom_notifier: A notification object that will dispatch events that - occur related to the atoms the engine contains. """ def __init__(self, flow, flow_detail, backend, options): @@ -46,25 +41,8 @@ class EngineBase(object): self._options = {} else: self._options = dict(options) - self._notifier = notifier.Notifier() - self._atom_notifier = notifier.Notifier() - - @property - def notifier(self): - """The flow notifier.""" - return self._notifier - - @property - @deprecation.moved_property('atom_notifier', version="0.6", - removal_version="?") - def task_notifier(self): - """The task notifier.""" - return self._atom_notifier - - @property - def atom_notifier(self): - """The atom notifier.""" - return self._atom_notifier + self.notifier = notifier.Notifier() + self.task_notifier = notifier.Notifier() @property def options(self): diff --git a/taskflow/tests/unit/test_failure.py b/taskflow/tests/unit/test_failure.py index b70add4f..793274e1 100644 --- a/taskflow/tests/unit/test_failure.py +++ b/taskflow/tests/unit/test_failure.py @@ -22,7 +22,6 @@ from taskflow import exceptions from taskflow import test from taskflow.tests import utils as test_utils from taskflow.types import failure -from taskflow.utils import misc def _captured_failure(msg): @@ -39,17 +38,6 @@ def _make_exc_info(msg): return sys.exc_info() -class DeprecatedTestCase(test.TestCase): - def test_deprecated(self): - f = None - try: - raise RuntimeError("Woot!") - except RuntimeError: - f = misc.Failure() - self.assertIsInstance(f, failure.Failure) - self.assertIsInstance(f, misc.Failure) - - class GeneralFailureObjTestsMixin(object): def test_captures_message(self): diff --git a/taskflow/tests/unit/test_notifier.py b/taskflow/tests/unit/test_notifier.py index 61dc7746..0761cb6e 100644 --- a/taskflow/tests/unit/test_notifier.py +++ b/taskflow/tests/unit/test_notifier.py @@ -20,15 +20,6 @@ import functools from taskflow import states from taskflow import test from taskflow.types import notifier as nt -from taskflow.utils import misc - - -class DeprecatedTestCase(test.TestCase): - def test_deprecated(self): - notifier = misc.Notifier() - self.assertIsInstance(notifier, misc.Notifier) - self.assertIsInstance(notifier, nt.Notifier) - self.assertTrue(hasattr(misc.Notifier, 'ANY')) class NotifierTest(test.TestCase): diff --git a/taskflow/utils/deprecation.py b/taskflow/utils/deprecation.py index b83cce58..899e035d 100644 --- a/taskflow/utils/deprecation.py +++ b/taskflow/utils/deprecation.py @@ -14,11 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. -import functools import warnings -import wrapt - from taskflow.utils import reflection @@ -27,67 +24,72 @@ def deprecation(message, stacklevel=2): warnings.warn(message, category=DeprecationWarning, stacklevel=stacklevel) -class MovedClassProxy(wrapt.ObjectProxy): - """Acts as a proxy to a class that was moved to another location.""" +# Helper accessors for the moved proxy (since it will not have easy access +# to its own getattr and setattr functions). +_setattr = object.__setattr__ +_getattr = object.__getattribute__ + + +class MovedClassProxy(object): + """Acts as a proxy to a class that was moved to another location. + + Partially based on: + + http://code.activestate.com/recipes/496741-object-proxying/ and other + various examination of how to make a good enough proxy for our usage to + move the various types we want to move during the deprecation process. + + And partially based on the wrapt object proxy (which we should just use + when it becomes available @ http://review.openstack.org/#/c/94754/). + """ + + __slots__ = [ + '__wrapped__', '__message__', '__stacklevel__', + # Ensure weakrefs can be made, + # https://docs.python.org/2/reference/datamodel.html#slots + '__weakref__', + ] def __init__(self, wrapped, message, stacklevel): - super(MovedClassProxy, self).__init__(wrapped) - self._self_message = message - self._self_stacklevel = stacklevel - - def __call__(self, *args, **kwargs): - deprecation(self._self_message, stacklevel=self._self_stacklevel) - return self.__wrapped__(*args, **kwargs) + # We can't assign to these directly, since we are overriding getattr + # and setattr and delattr so we have to do this hoop jump to ensure + # that we don't invoke those methods (and cause infinite recursion). + _setattr(self, '__wrapped__', wrapped) + _setattr(self, '__message__', message) + _setattr(self, '__stacklevel__', stacklevel) + try: + _setattr(self, '__qualname__', wrapped.__qualname__) + except AttributeError: + pass def __instancecheck__(self, instance): - deprecation(self._self_message, stacklevel=self._self_stacklevel) - return isinstance(instance, self.__wrapped__) + deprecation( + _getattr(self, '__message__'), _getattr(self, '__stacklevel__')) + return isinstance(instance, _getattr(self, '__wrapped__')) def __subclasscheck__(self, instance): - deprecation(self._self_message, stacklevel=self._self_stacklevel) - return issubclass(instance, self.__wrapped__) + deprecation( + _getattr(self, '__message__'), _getattr(self, '__stacklevel__')) + return issubclass(instance, _getattr(self, '__wrapped__')) + def __call__(self, *args, **kwargs): + deprecation( + _getattr(self, '__message__'), _getattr(self, '__stacklevel__')) + return _getattr(self, '__wrapped__')(*args, **kwargs) -def _generate_moved_message(kind, old_name, new_name, - message=None, version=None, removal_version=None): - message_components = [ - "%s '%s' has moved to '%s'" % (kind, old_name, new_name), - ] - if version: - message_components.append(" in version '%s'" % version) - if removal_version: - if removal_version == "?": - message_components.append(" and will be removed in a future" - " version") - else: - message_components.append(" and will be removed in version" - " '%s'" % removal_version) - if message: - message_components.append(": %s" % message) - return ''.join(message_components) + def __getattribute__(self, name): + return getattr(_getattr(self, '__wrapped__'), name) + def __setattr__(self, name, value): + setattr(_getattr(self, '__wrapped__'), name, value) -def _moved_decorator(kind, new_name, message=None, - version=None, removal_version=None): - """Decorates a method/function/other that was moved to another location.""" + def __delattr__(self, name): + delattr(_getattr(self, '__wrapped__'), name) - @wrapt.decorator - def decorator(wrapped, instance, args, kwargs): - try: - old_name = wrapped.__qualname__ - except AttributeError: - old_name = wrapped.__name__ - out_message = _generate_moved_message(kind, old_name, new_name, - message=message, version=version, - removal_version=removal_version) - deprecation(out_message, 3) - return wrapped(*args, **kwargs) - - return decorator - - -"""Decorates a property that was moved to another location.""" -moved_property = functools.partial(_moved_decorator, 'Property') + def __repr__(self): + wrapped = _getattr(self, '__wrapped__') + return "<%s at 0x%x for %r at 0x%x>" % ( + type(self).__name__, id(self), wrapped, id(wrapped)) def moved_class(new_class, old_class_name, old_module_name, message=None, @@ -99,7 +101,18 @@ def moved_class(new_class, old_class_name, old_module_name, message=None, """ old_name = ".".join((old_module_name, old_class_name)) new_name = reflection.get_class_name(new_class) - out_message = _generate_moved_message('Class', old_name, new_name, - message=message, version=version, - removal_version=removal_version) - return MovedClassProxy(new_class, out_message, 3) + message_components = [ + "Class '%s' has moved to '%s'" % (old_name, new_name), + ] + if version: + message_components.append(" in version '%s'" % version) + if removal_version: + if removal_version == "?": + message_components.append(" and will be removed in a future" + " version") + else: + message_components.append(" and will be removed in version '%s'" + % removal_version) + if message: + message_components.append(": %s" % message) + return MovedClassProxy(new_class, "".join(message_components), 3) From 487cc51832982b4b64d4bde3dd1e112628bcb78e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 20 Nov 2014 16:26:08 -0800 Subject: [PATCH 102/240] Mark 'task_notifier' as renamed to 'atom_notifier' Deprecate the usage of an engines 'task_notifier' in favor of the more appropriately named 'atom_notifier' and mark the 'task_notifier' property as subject to removal in a future version. This makes the usage of this notifier more clear since it is not only used for task notification but also for retry notification (which is why naming it atom notifier is more suited to its actual usage). Change-Id: I79d58a08fd8e6d7c8990e70bdfaae415202aa929 --- taskflow/engines/action_engine/engine.py | 2 +- taskflow/engines/action_engine/runtime.py | 8 +-- taskflow/engines/base.py | 26 ++++++++- taskflow/utils/deprecation.py | 68 +++++++++++++++++++---- 4 files changed, 86 insertions(+), 18 deletions(-) diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index 730d2895..5a0fc9fb 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -217,7 +217,7 @@ class ActionEngine(base.EngineBase): self._compilation = self._compiler.compile() self._runtime = runtime.Runtime(self._compilation, self.storage, - self.task_notifier, + self.atom_notifier, self._task_executor) self._compiled = True diff --git a/taskflow/engines/action_engine/runtime.py b/taskflow/engines/action_engine/runtime.py index 06959f29..9e053a47 100644 --- a/taskflow/engines/action_engine/runtime.py +++ b/taskflow/engines/action_engine/runtime.py @@ -36,8 +36,8 @@ class Runtime(object): action engine to run to completion. """ - def __init__(self, compilation, storage, task_notifier, task_executor): - self._task_notifier = task_notifier + def __init__(self, compilation, storage, atom_notifier, task_executor): + self._atom_notifier = atom_notifier self._task_executor = task_executor self._storage = storage self._compilation = compilation @@ -68,7 +68,7 @@ class Runtime(object): @misc.cachedproperty def retry_action(self): - return ra.RetryAction(self._storage, self._task_notifier, + return ra.RetryAction(self._storage, self._atom_notifier, lambda atom: sc.ScopeWalker(self.compilation, atom, names_only=True)) @@ -76,7 +76,7 @@ class Runtime(object): @misc.cachedproperty def task_action(self): return ta.TaskAction(self._storage, self._task_executor, - self._task_notifier, + self._atom_notifier, lambda atom: sc.ScopeWalker(self.compilation, atom, names_only=True)) diff --git a/taskflow/engines/base.py b/taskflow/engines/base.py index 2fab9d0f..2c409726 100644 --- a/taskflow/engines/base.py +++ b/taskflow/engines/base.py @@ -20,6 +20,7 @@ import abc import six from taskflow.types import notifier +from taskflow.utils import deprecation from taskflow.utils import misc @@ -31,6 +32,10 @@ class EngineBase(object): occur related to the flow the engine contains. :ivar task_notifier: A notification object that will dispatch events that occur related to the tasks the engine contains. + occur related to the tasks the engine + contains (deprecated). + :ivar atom_notifier: A notification object that will dispatch events that + occur related to the atoms the engine contains. """ def __init__(self, flow, flow_detail, backend, options): @@ -41,8 +46,25 @@ class EngineBase(object): self._options = {} else: self._options = dict(options) - self.notifier = notifier.Notifier() - self.task_notifier = notifier.Notifier() + self._notifier = notifier.Notifier() + self._atom_notifier = notifier.Notifier() + + @property + def notifier(self): + """The flow notifier.""" + return self._notifier + + @property + @deprecation.moved_property('atom_notifier', version="0.6", + removal_version="?") + def task_notifier(self): + """The task notifier.""" + return self._atom_notifier + + @property + def atom_notifier(self): + """The atom notifier.""" + return self._atom_notifier @property def options(self): diff --git a/taskflow/utils/deprecation.py b/taskflow/utils/deprecation.py index 899e035d..7e8e60bc 100644 --- a/taskflow/utils/deprecation.py +++ b/taskflow/utils/deprecation.py @@ -14,8 +14,11 @@ # License for the specific language governing permissions and limitations # under the License. +import functools import warnings +import six + from taskflow.utils import reflection @@ -92,17 +95,10 @@ class MovedClassProxy(object): type(self).__name__, id(self), wrapped, id(wrapped)) -def moved_class(new_class, old_class_name, old_module_name, message=None, - version=None, removal_version=None): - """Deprecates a class that was moved to another location. - - This will emit warnings when the old locations class is initialized, - telling where the new and improved location for the old class now is. - """ - old_name = ".".join((old_module_name, old_class_name)) - new_name = reflection.get_class_name(new_class) +def _generate_moved_message(kind, old_name, new_name, + message=None, version=None, removal_version=None): message_components = [ - "Class '%s' has moved to '%s'" % (old_name, new_name), + "%s '%s' has moved to '%s'" % (kind, old_name, new_name), ] if version: message_components.append(" in version '%s'" % version) @@ -115,4 +111,54 @@ def moved_class(new_class, old_class_name, old_module_name, message=None, % removal_version) if message: message_components.append(": %s" % message) - return MovedClassProxy(new_class, "".join(message_components), 3) + return ''.join(message_components) + + +def _moved_decorator(kind, new_attribute_name, message=None, + version=None, removal_version=None): + """Decorates a method/property that was moved to another location.""" + + def decorator(f): + try: + old_attribute_name = f.__qualname__ + fully_qualified = True + except AttributeError: + old_attribute_name = f.__name__ + fully_qualified = False + + @six.wraps(f) + def wrapper(self, *args, **kwargs): + base_name = reflection.get_class_name(self, fully_qualified=False) + if fully_qualified: + old_name = old_attribute_name + else: + old_name = ".".join((base_name, old_attribute_name)) + new_name = ".".join((base_name, new_attribute_name)) + out_message = _generate_moved_message( + kind, old_name=old_name, new_name=new_name, message=message, + version=version, removal_version=removal_version) + deprecation(out_message, 3) + return f(self, *args, **kwargs) + + return wrapper + + return decorator + + +"""Decorates a *instance* property that was moved to another location.""" +moved_property = functools.partial(_moved_decorator, 'Property') + + +def moved_class(new_class, old_class_name, old_module_name, message=None, + version=None, removal_version=None): + """Deprecates a class that was moved to another location. + + This will emit warnings when the old locations class is initialized, + telling where the new and improved location for the old class now is. + """ + old_name = ".".join((old_module_name, old_class_name)) + new_name = reflection.get_class_name(new_class) + out_message = _generate_moved_message('Class', old_name, new_name, + message=message, version=version, + removal_version=removal_version) + return MovedClassProxy(new_class, out_message, 3) From 17fb4b456fbe0c68b42e86449d9054d9a4005dd9 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 19 Nov 2014 17:53:43 -0800 Subject: [PATCH 103/240] Add a jobboard high level architecture diagram Incorporate a diagram into the documentation which shows at a high level what the different components of a jobboard are and the different actors that are typically involved in using it. Also moves the graffle (omnigraffle) xml/text files to be tarballs; to save space and to avoid false line counts in gerrit and stackalytics. Change-Id: I5ddb0bdb7e2eb39844d064546a34795f08cf0c1f --- doc/diagrams/core.graffle | 8023 ----------------------------- doc/diagrams/core.graffle.tgz | Bin 0 -> 16344 bytes doc/diagrams/jobboard.graffle.tgz | Bin 0 -> 27254 bytes doc/source/img/jobboard.png | Bin 0 -> 110260 bytes doc/source/jobs.rst | 7 + 5 files changed, 7 insertions(+), 8023 deletions(-) delete mode 100644 doc/diagrams/core.graffle create mode 100644 doc/diagrams/core.graffle.tgz create mode 100644 doc/diagrams/jobboard.graffle.tgz create mode 100644 doc/source/img/jobboard.png diff --git a/doc/diagrams/core.graffle b/doc/diagrams/core.graffle deleted file mode 100644 index a570fe59..00000000 --- a/doc/diagrams/core.graffle +++ /dev/null @@ -1,8023 +0,0 @@ - - - - - ActiveLayerIndex - 0 - ApplicationVersion - - com.omnigroup.OmniGrafflePro - 139.18.0.187838 - - AutoAdjust - - BackgroundGraphic - - Bounds - {{0, 0}, {1152, 2199}} - Class - SolidGraphic - ID - 2 - Style - - shadow - - Draws - NO - - stroke - - Draws - NO - - - - BaseZoom - 0 - CanvasOrigin - {0, 0} - ColumnAlign - 1 - ColumnSpacing - 36 - CreationDate - 2014-07-08 20:47:01 +0000 - Creator - Joshua Harlow - DisplayScale - 1 0/72 in = 1.0000 in - ExportShapes - - - InspectorGroup - 255 - ShapeImageRect - {{2, 2}, {22, 22}} - ShapeName - 33C70F48-B008-4466-BD81-E84D73C055CA-438-0000056AF6035FFB - ShouldExport - YES - StrokePath - - elements - - - element - MOVETO - point - {0.40652500000000003, 0.088786000000000004} - - - control1 - {0.39769700000000002, -0.059801} - control2 - {0.312282, -0.20657200000000001} - element - CURVETO - point - {0.15027599999999999, -0.32002000000000003} - - - control1 - {-0.028644599999999999, -0.44531500000000002} - control2 - {-0.26560600000000001, -0.50519099999999995} - element - CURVETO - point - {-0.5, -0.49964799999999998} - - - element - LINETO - point - {-0.5, -0.25638699999999998} - - - control1 - {-0.358902, -0.262291} - control2 - {-0.21507999999999999, -0.22622900000000001} - element - CURVETO - point - {-0.10728, -0.148201} - - - control1 - {-0.0160971, -0.082201999999999997} - control2 - {0.033605599999999999, 0.0024510600000000001} - element - CURVETO - point - {0.041826200000000001, 0.088786000000000004} - - - element - LINETO - point - {-0.043046000000000001, 0.088786000000000004} - - - element - LINETO - point - {0.22847700000000001, 0.5} - - - element - LINETO - point - {0.5, 0.088786000000000004} - - - element - LINETO - point - {0.40652500000000003, 0.088786000000000004} - - - element - CLOSE - - - element - MOVETO - point - {0.40652500000000003, 0.088786000000000004} - - - - TextBounds - {{0, 0}, {1, 1}} - - - GraphDocumentVersion - 8 - GraphicsList - - - Class - LineGraphic - ID - 1169 - Points - - {148.34850886899716, 1297.778564453125} - {148.34850886899716, 1565.8355233257191} - - Style - - stroke - - HeadArrow - 0 - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - 0 - - - - - Bounds - {{108.29570600619962, 1459.9910998882619}, {30, 14}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica - Size - 12 - - ID - 1167 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\fs24 \cf0 Emits} - VerticalPad - 0 - - Wrap - NO - - - Class - LineGraphic - ID - 1166 - Points - - {172.01007495190493, 1463.7899284362793} - {108.29570625246291, 1489.3205401648188} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - 0 - - - - - Bounds - {{28, 1489.3205331673671}, {108.00000616531918, 60.376010894775391}} - Class - ShapedGraphic - ID - 1165 - Shape - Cloud - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Board\ -Notifications} - VerticalPad - 0 - - - - Class - LineGraphic - ID - 1161 - Points - - {16.938813712387287, 1214.6957778930664} - {550.61227271339396, 1214.6957778930664} - - Style - - stroke - - HeadArrow - 0 - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - 0 - - - - - Class - Group - Graphics - - - Bounds - {{177.05329513549805, 1254.1663719071589}, {82, 22}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 1163 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\b\fs36 \cf0 (optional)} - VerticalPad - 0 - - Wrap - NO - - - Bounds - {{56.053289698640896, 1231.7786193741999}, {116, 66}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica-BoldOblique - Size - 18 - - ID - 1164 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\b\fs36 \cf0 Posting & \ -Consumption\ -Phase} - VerticalPad - 0 - - Wrap - NO - - - ID - 1162 - - - Bounds - {{560.82414838901218, 1484.9621440334549}, {71, 28}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica - Size - 12 - - ID - 1159 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\fs24 \cf0 Consumption\ -Loop} - VerticalPad - 0 - - Wrap - NO - - - Class - Group - Graphics - - - Bounds - {{521.16725664085266, 1494.1828820625792}, {27.016406012875592, 38.542124503311257}} - Class - ShapedGraphic - ID - 1155 - Magnets - - {0.15027599999999999, -0.32002000000000003} - {-0.5, -0.49964799999999998} - {-0.5, -0.25638699999999998} - {-0.10728, -0.148201} - {0.041826500000000003, 0.088786000000000004} - {-0.043045800000000002, 0.088786000000000004} - {0.22847700000000001, 0.5} - {0.5, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - - Rotation - 90 - Shape - Bezier - ShapeData - - UnitPoints - - {0.406219, 0.101163} - {0.39736100000000002, -0.042958700000000002} - {0.31166700000000003, -0.185311} - {0.149117, -0.29534500000000002} - {-0.030395499999999999, -0.41686299999999998} - {-0.261517, -0.50514099999999995} - {-0.49668800000000002, -0.49976700000000002} - {-0.496693, -0.49976500000000001} - {-0.062913899999999995, -0.36058899999999999} - {-0.062913899999999995, -0.36058899999999999} - {-0.062918699999999994, -0.36058899999999999} - {-0.5, -0.21609700000000001} - {-0.5, -0.21609600000000001} - {-0.35843000000000003, -0.22182399999999999} - {-0.217449, -0.204378} - {-0.10928400000000001, -0.12870000000000001} - {-0.017806099999999998, -0.064687300000000003} - {0.032062500000000001, 0.0174179} - {0.040309900000000003, 0.101163} - {0.040309900000000003, 0.101163} - {-0.044847499999999998, 0.101163} - {-0.044847499999999998, 0.101163} - {-0.044847499999999998, 0.101163} - {0.22758200000000001, 0.5} - {0.22758200000000001, 0.5} - {0.22758200000000001, 0.5} - {0.5, 0.101163} - {0.5, 0.101163} - {0.5, 0.101163} - {0.406219, 0.101163} - - - - - Bounds - {{501.82414838901218, 1486.2594805049087}, {26.999999999999996, 38.288223134554855}} - Class - ShapedGraphic - ID - 1156 - Magnets - - {0.15027599999999999, -0.32002000000000003} - {-0.5, -0.49964799999999998} - {-0.5, -0.25638699999999998} - {-0.10728, -0.148201} - {0.041826500000000003, 0.088786000000000004} - {-0.043045800000000002, 0.088786000000000004} - {0.22847700000000001, 0.5} - {0.5, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - - Rotation - 180 - Shape - 33C70F48-B008-4466-BD81-E84D73C055CA-438-0000056AF6035FFB - - - Bounds - {{509.94240895873349, 1465.3378642343948}, {27.016406012875589, 38.264972185430459}} - Class - ShapedGraphic - ID - 1157 - Magnets - - {0.15027599999999999, -0.32002000000000003} - {-0.5, -0.49964799999999998} - {-0.5, -0.25638699999999998} - {-0.10728, -0.148201} - {0.041826500000000003, 0.088786000000000004} - {-0.043045800000000002, 0.088786000000000004} - {0.22847700000000001, 0.5} - {0.5, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - - Rotation - 270 - Shape - 33C70F48-B008-4466-BD81-E84D73C055CA-438-0000056AF6035FFB - - - Bounds - {{528.82414838901218, 1473.3774855848621}, {27.000000000000004, 38.288223134554862}} - Class - ShapedGraphic - ID - 1158 - Magnets - - {0.15027599999999999, -0.32002000000000003} - {-0.5, -0.49964799999999998} - {-0.5, -0.25638699999999998} - {-0.10728, -0.148201} - {0.041826500000000003, 0.088786000000000004} - {-0.043045800000000002, 0.088786000000000004} - {0.22847700000000001, 0.5} - {0.5, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - - Shape - 33C70F48-B008-4466-BD81-E84D73C055CA-438-0000056AF6035FFB - - - ID - 1154 - - - Class - Group - Graphics - - - Bounds - {{302.01802465549162, 1480.6049629105769}, {37, 28}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 1150 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Align - 0 - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural - -\f0\fs24 \cf0 - wait()\ -....} - VerticalPad - 0 - - Wrap - NO - - - Bounds - {{230.34967062106779, 1480.6049629105769}, {63, 42}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 1151 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Align - 0 - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural - -\f0\fs24 \cf0 - abandon()\ -- iterjobs()\ -} - VerticalPad - 0 - - Wrap - NO - - - Bounds - {{177.68130514255216, 1480.6049629105769}, {44, 42}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 1152 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Align - 0 - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural - -\f0\fs24 \cf0 - post()\ -- claim()\ -} - VerticalPad - 0 - - Wrap - NO - - - Bounds - {{170.82414838901212, 1475.3192573441706}, {175.99597549438477, 38.571445465087891}} - Class - ShapedGraphic - ID - 1153 - Shape - Rectangle - Style - - stroke - - Pattern - 1 - - - - - ID - 1149 - - - Class - LineGraphic - ID - 1148 - Points - - {290.10982343784025, 1540.438820465416} - {289.2121722540532, 1513.7478166474543} - - Style - - stroke - - HeadArrow - UMLInheritance - Legacy - - LineType - 1 - TailArrow - 0 - - - Tail - - ID - 1147 - - - - Bounds - {{245.10982343784025, 1540.438820465416}, {90, 36}} - Class - ShapedGraphic - ID - 1147 - Magnets - - {0, 1} - {0, -1} - {1, 0} - {-1, 0} - - Shape - Rectangle - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\i\b\fs24 \cf0 Zookeeper\ -Jobboard} - VerticalPad - 0 - - - - Bounds - {{435.40182134738615, 1470.9621696472168}, {58, 56}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica - Size - 12 - - ID - 1146 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Align - 0 - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural - -\f0\i\fs24 \cf0 - Claim job\ -- Load job\ -- Translate\ -- Activate} - VerticalPad - 0 - - Wrap - NO - - - Class - Group - Graphics - - - Class - Group - Graphics - - - Bounds - {{539.9018270694321, 1424.6444211854182}, {33, 12}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 1117 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs20 \cf0 Worker} - VerticalPad - 0 - - Wrap - NO - - - Class - Group - Graphics - - - Bounds - {{530.40181944003757, 1370.6049924744807}, {52, 72}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 1119 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs20 \cf0 Specialized\ -\ -\ -\ -\ -} - VerticalPad - 0 - - Wrap - NO - - - Class - Group - Graphics - - - Bounds - {{545.72859289858161, 1420.3446371019186}, {19.299808229718884, 19.299808879032174}} - Class - ShapedGraphic - ID - 1121 - Shape - Rectangle - Style - - fill - - Color - - b - 0.4 - g - 1 - r - 1 - - Draws - NO - FillType - 2 - GradientAngle - 90 - GradientColor - - b - 0.4 - g - 1 - r - 1 - - MiddleColor - - b - 0.4 - g - 1 - r - 1 - - TrippleBlend - YES - - shadow - - Beneath - YES - Draws - NO - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - Draws - NO - Width - 1.5 - - - VFlip - YES - Wrap - NO - - - Class - LineGraphic - ID - 1122 - Points - - {545.72859289858161, 1402.9748103444847} - {565.02840112830063, 1402.9748103444847} - {565.02840112830063, 1402.9748103444847} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 1123 - Points - - {555.37849701344112, 1410.6947336363717} - {545.7285925739252, 1420.3446377512312} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 1124 - Points - - {555.37849701344112, 1410.6947336363719} - {565.02840112830063, 1420.7306576742712} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 1125 - Points - - {555.37849701344112, 1397.1848678755687} - {555.37849701344112, 1410.6947352596553} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Bounds - {{549.58855454452544, 1385.6049829377375}, {11.57988428851805, 11.57988428851805}} - Class - ShapedGraphic - ID - 1126 - Shape - Circle - Style - - fill - - Color - - b - 0.4 - g - 1 - r - 1 - - Draws - NO - FillType - 2 - GradientAngle - 90 - GradientColor - - b - 0.4 - g - 1 - r - 1 - - MiddleColor - - b - 0.4 - g - 1 - r - 1 - - TrippleBlend - YES - - shadow - - Beneath - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - Width - 1.5 - - - - - ID - 1120 - - - ID - 1118 - - - ID - 1116 - - - Class - Group - Graphics - - - Bounds - {{492.40181944003757, 1394.5655408753596}, {47, 12}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 1128 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs20 \cf0 Conductor} - VerticalPad - 0 - - Wrap - NO - - - Class - Group - Graphics - - - Bounds - {{505.22859289858167, 1444.305185502797}, {19.299808229718884, 19.299808879032174}} - Class - ShapedGraphic - ID - 1130 - Shape - Rectangle - Style - - fill - - Color - - b - 0.4 - g - 1 - r - 1 - - Draws - NO - FillType - 2 - GradientAngle - 90 - GradientColor - - b - 0.4 - g - 1 - r - 1 - - MiddleColor - - b - 0.4 - g - 1 - r - 1 - - TrippleBlend - YES - - shadow - - Beneath - YES - Draws - NO - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - Draws - NO - Width - 1.5 - - - VFlip - YES - Wrap - NO - - - Class - LineGraphic - ID - 1131 - Points - - {505.22859289858161, 1426.9353587453638} - {524.52840112830063, 1426.9353587453638} - {524.52840112830063, 1426.9353587453638} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 1132 - Points - - {514.87849701344112, 1434.6552820372506} - {505.2285925739252, 1444.3051861521101} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 1133 - Points - - {514.87849701344112, 1434.6552820372508} - {524.52840112830063, 1444.6912060751501} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 1134 - Points - - {514.87849701344112, 1421.1454162764476} - {514.87849701344112, 1434.6552836605342} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Bounds - {{509.08855454452544, 1409.5655313386164}, {11.57988428851805, 11.57988428851805}} - Class - ShapedGraphic - ID - 1135 - Shape - Circle - Style - - fill - - Color - - b - 0.4 - g - 1 - r - 1 - - Draws - NO - FillType - 2 - GradientAngle - 90 - GradientColor - - b - 0.4 - g - 1 - r - 1 - - MiddleColor - - b - 0.4 - g - 1 - r - 1 - - TrippleBlend - YES - - shadow - - Beneath - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - Width - 1.5 - - - - - ID - 1129 - - - ID - 1127 - - - Class - Group - Graphics - - - Bounds - {{454.40181944003757, 1373.5655408753596}, {47, 12}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 1137 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs20 \cf0 Conductor} - VerticalPad - 0 - - Wrap - NO - - - Class - Group - Graphics - - - Bounds - {{467.22859289858161, 1423.305185502797}, {19.299808229718884, 19.299808879032174}} - Class - ShapedGraphic - ID - 1139 - Shape - Rectangle - Style - - fill - - Color - - b - 0.4 - g - 1 - r - 1 - - Draws - NO - FillType - 2 - GradientAngle - 90 - GradientColor - - b - 0.4 - g - 1 - r - 1 - - MiddleColor - - b - 0.4 - g - 1 - r - 1 - - TrippleBlend - YES - - shadow - - Beneath - YES - Draws - NO - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - Draws - NO - Width - 1.5 - - - VFlip - YES - Wrap - NO - - - Class - LineGraphic - ID - 1140 - Points - - {467.22859289858161, 1405.9353587453638} - {486.52840112830063, 1405.9353587453638} - {486.52840112830063, 1405.9353587453638} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 1141 - Points - - {476.87849701344112, 1413.6552820372506} - {467.2285925739252, 1423.3051861521101} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 1142 - Points - - {476.87849701344112, 1413.6552820372508} - {486.52840112830063, 1423.6912060751501} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 1143 - Points - - {476.87849701344112, 1400.1454162764476} - {476.87849701344112, 1413.6552836605342} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Bounds - {{471.08855454452544, 1388.5655313386164}, {11.57988428851805, 11.57988428851805}} - Class - ShapedGraphic - ID - 1144 - Shape - Circle - Style - - fill - - Color - - b - 0.4 - g - 1 - r - 1 - - Draws - NO - FillType - 2 - GradientAngle - 90 - GradientColor - - b - 0.4 - g - 1 - r - 1 - - MiddleColor - - b - 0.4 - g - 1 - r - 1 - - TrippleBlend - YES - - shadow - - Beneath - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - Width - 1.5 - - - - - ID - 1138 - - - ID - 1136 - - - Bounds - {{428.40181944003757, 1349.4199200524531}, {175.99597549438477, 117.33065032958984}} - Class - ShapedGraphic - ID - 1145 - Shape - Cloud - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\fs24 \cf0 Workers} - VerticalPad - 0 - - TextPlacement - 0 - TextRelativeArea - {{0.14999999999999999, -0.15000001192092893}, {0.69999999999999996, 0.69999999999999996}} - - - ID - 1115 - - - Class - LineGraphic - Head - - ID - 968 - - ID - 1065 - Points - - {440.82414838901212, 1414.6049619569026} - {387.27826521030119, 1414.6049619569026} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - TailArrow - FilledArrow - - - - - Bounds - {{93.246466708152184, 1417.5425142326339}, {50, 42}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 996 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Receives\ -Job\ -} - VerticalPad - 0 - - Wrap - NO - - - Bounds - {{91.82415492531527, 1366.6049531974777}, {50, 42}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 995 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Posts\ -Workflow\ -} - VerticalPad - 0 - - Wrap - NO - - - Class - LineGraphic - ID - 994 - Points - - {110.33829994431926, 1405.0729529199277} - {164.08388984446532, 1405.5} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - TailArrow - FilledArrow - - - - - Class - Group - Graphics - - - Bounds - {{50.1444289081536, 1351.4198986563467}, {35, 28}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 986 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 GillSans;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Library\ -User} - VerticalPad - 0 - - Wrap - NO - - - Class - Group - Graphics - - - Bounds - {{53.471203982158727, 1435.4435211691641}, {28.346457481384277, 28.346458435058594}} - Class - ShapedGraphic - ID - 988 - Shape - Rectangle - Style - - fill - - Color - - b - 0.4 - g - 1 - r - 1 - - Draws - NO - FillType - 2 - GradientAngle - 90 - GradientColor - - b - 0.4 - g - 1 - r - 1 - - MiddleColor - - b - 0.4 - g - 1 - r - 1 - - TrippleBlend - YES - - shadow - - Beneath - YES - Draws - NO - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - Draws - NO - Width - 1.5 - - - VFlip - YES - Wrap - NO - - - Class - LineGraphic - ID - 989 - Points - - {53.471203982158727, 1409.9317103895926} - {81.817661463543004, 1409.9317103895926} - {81.817661463543004, 1409.9317103895926} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 990 - Points - - {67.644432722850866, 1421.2702933821463} - {53.471203505321569, 1435.4435221228384} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 991 - Points - - {67.644432722850866, 1421.2702933821463} - {81.817661463543004, 1436.0104861675161} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 992 - Points - - {67.644432722850866, 1401.4277731451773} - {67.644432722850866, 1421.2702957663321} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Bounds - {{59.140495478435582, 1384.4198986563467}, {17.00787353515625, 17.00787353515625}} - Class - ShapedGraphic - ID - 993 - Shape - Circle - Style - - fill - - Color - - b - 0.4 - g - 1 - r - 1 - - Draws - NO - FillType - 2 - GradientAngle - 90 - GradientColor - - b - 0.4 - g - 1 - r - 1 - - MiddleColor - - b - 0.4 - g - 1 - r - 1 - - TrippleBlend - YES - - shadow - - Beneath - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - Width - 1.5 - - - - - ID - 987 - - - ID - 1001 - - - Bounds - {{237.44151899448087, 1414.6049619569026}, {54, 36}} - Class - ShapedGraphic - ID - 961 - Shape - Rectangle - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Job} - - - - Bounds - {{228.44151899448087, 1405.6049619569026}, {54, 36}} - Class - ShapedGraphic - ID - 1000 - Shape - Rectangle - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Job1} - - - - Class - Group - Graphics - - - Class - LineGraphic - Head - - ID - 968 - - ID - 967 - Points - - {273.94151899473252, 1414.6017734596319} - {332.27826523991331, 1414.5950095848621} - - Style - - stroke - - HeadArrow - 0 - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - FilledArrow - - - Tail - - ID - 969 - - - - Bounds - {{332.77826521030119, 1396.6049619569026}, {54, 36}} - Class - ShapedGraphic - ID - 968 - Shape - Rectangle - Style - - stroke - - Pattern - 1 - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Owner} - - - - Bounds - {{219.44151899448087, 1396.6049619569026}, {54, 36}} - Class - ShapedGraphic - ID - 969 - Shape - Rectangle - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Job1} - - - - ID - 966 - - - Class - Group - Graphics - - - Class - LineGraphic - Head - - ID - 972 - - ID - 971 - Points - - {264.94151899473252, 1405.6017734596319} - {323.27826523991331, 1405.5950095848621} - - Style - - stroke - - HeadArrow - 0 - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - FilledArrow - - - Tail - - ID - 973 - - - - Bounds - {{323.77826521030119, 1387.6049619569026}, {54, 36}} - Class - ShapedGraphic - ID - 972 - Shape - Rectangle - Style - - stroke - - Pattern - 1 - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Owner} - - - - Bounds - {{210.44151899448087, 1387.6049619569026}, {54, 36}} - Class - ShapedGraphic - ID - 973 - Shape - Rectangle - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Job1} - - - - ID - 970 - - - Class - Group - Graphics - - - Class - LineGraphic - Head - - ID - 976 - - ID - 975 - Points - - {255.94151899473252, 1396.6017734596319} - {314.27826523991337, 1396.5950095848621} - - Style - - stroke - - HeadArrow - 0 - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - FilledArrow - - - Tail - - ID - 977 - - - - Bounds - {{314.77826521030119, 1378.6049619569026}, {54, 36}} - Class - ShapedGraphic - ID - 976 - Shape - Rectangle - Style - - stroke - - Pattern - 1 - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Owner} - - - - Bounds - {{201.44151899448087, 1378.6049619569026}, {54, 36}} - Class - ShapedGraphic - ID - 977 - Shape - Rectangle - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Job1} - - - - ID - 974 - - - Class - Group - Graphics - - - Class - LineGraphic - Head - - ID - 980 - - ID - 979 - Points - - {246.94151899473252, 1387.6017734596319} - {305.27826523991337, 1387.5950095848621} - - Style - - stroke - - HeadArrow - 0 - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - FilledArrow - - - Tail - - ID - 981 - - - - Bounds - {{305.77826521030119, 1369.6049619569026}, {54, 36}} - Class - ShapedGraphic - ID - 980 - Shape - Rectangle - Style - - stroke - - Pattern - 1 - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Owner} - - - - Bounds - {{192.44151899448087, 1369.6049619569026}, {54, 36}} - Class - ShapedGraphic - ID - 981 - Shape - Rectangle - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Job1} - - - - ID - 978 - - - Bounds - {{170.82414838901212, 1345.6049695862971}, {236.99999999999997, 168}} - Class - ShapedGraphic - FitText - Vertical - Flow - Resize - ID - 983 - Shape - Rectangle - Style - - fill - - GradientCenter - {-0.29411799999999999, -0.264706} - - - Text - - Align - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 \ -\ -\ -\ -\ -\ -\ -\ -\ -\ -\ -} - VerticalPad - 0 - - TextPlacement - 0 - - - Bounds - {{170.82414838901212, 1331.6049695862971}, {236.99999999999997, 14}} - Class - ShapedGraphic - FitText - Vertical - Flow - Resize - ID - 984 - Shape - Rectangle - Style - - fill - - GradientCenter - {-0.29411799999999999, -0.264706} - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\i\b\fs24 \cf0 Jobboard} - VerticalPad - 0 - - TextPlacement - 0 - - - Class - LineGraphic - ID - 861 - Points - - {470.1300977351811, 156.79728666398489} - {409.22449458705552, 177.09915438002673} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - 0 - - - - - Bounds - {{476.1300977351811, 138.79728666398486}, {41, 28}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica - Size - 12 - - ID - 860 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\fs24 \cf0 Nested\ -subflow} - VerticalPad - 0 - - Wrap - NO - - - Class - LineGraphic - ID - 859 - Points - - {382.65871206690008, 221.8325309753418} - {382.65871206690008, 249.83253047325724} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - TailArrow - 0 - - - - - Bounds - {{359.27167431166708, 255.11224365234375}, {47, 47}} - Class - ShapedGraphic - HFlip - YES - ID - 855 - Shape - Circle - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Flow} - VerticalPad - 0 - - - - Bounds - {{355.77167171239853, 251.61224365234375}, {54, 54}} - Class - ShapedGraphic - HFlip - YES - ID - 856 - Shape - Circle - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Retry} - VerticalPad - 0 - - TextPlacement - 0 - TextRelativeArea - {{0.099999999999999978, 1.0000000238418578}, {0.80000000000000004, 0.69999999999999996}} - TextRotation - 305.1478271484375 - - - Bounds - {{290.73464965820312, 1032.5300847720423}, {27, 28}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica - Size - 12 - - ID - 839 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\fs24 \cf0 Run\ -Loop} - VerticalPad - 0 - - Wrap - NO - - - Class - LineGraphic - ID - 838 - Points - - {16.938772201538086, 784.51440811157227} - {550.61223120254476, 784.51440811157227} - - Style - - stroke - - HeadArrow - 0 - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - 0 - - - - - Bounds - {{478.53062537152402, 1122.9011524936079}, {63.714366912841797, 31.333333333333332}} - Class - ShapedGraphic - ID - 837 - Shape - Rectangle - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs20 \cf0 Completer} - VerticalPad - 0 - - - - Bounds - {{478.53062537152402, 1079.8705891391157}, {63.714366912841797, 31.333333333333332}} - Class - ShapedGraphic - ID - 836 - Shape - Rectangle - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs20 \cf0 Scheduler} - VerticalPad - 0 - - - - Bounds - {{372.92606544494629, 1123.8570556640625}, {61, 56}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 834 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Align - 0 - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural - -\f0\fs24 \cf0 - run()\ -- suspend()\ -...\ -} - VerticalPad - 0 - - Wrap - NO - - - Bounds - {{390.8163413254731, 1078.5120424153899}, {63.714366912841797, 31.333333333333332}} - Class - ShapedGraphic - ID - 832 - Shape - Rectangle - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs20 \cf0 Compiler} - VerticalPad - 0 - - - - Bounds - {{209.22450065612793, 852.73572444915771}, {80, 28}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica - Size - 12 - - ID - 831 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\fs24 \cf0 States, results,\ -progress...} - VerticalPad - 0 - - Wrap - NO - - - Bounds - {{195.91839599609375, 1080.2736424160523}, {156, 70}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica - Size - 12 - - ID - 828 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Align - 0 - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural - -\f0\i\fs24 \cf0 - PENDING -> RUNNING\ -- RUNNING -> SUCCESS\ -- SUSPENDED -> RUNNING\ -- FAILURE -> REVERTING\ -....} - VerticalPad - 0 - - Wrap - NO - - - Bounds - {{179.01475524902344, 1044.5300637912073}, {30, 14}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica - Size - 12 - - ID - 827 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\fs24 \cf0 Emits} - VerticalPad - 0 - - Wrap - NO - - - Class - LineGraphic - ID - 826 - Points - - {228.92846501504124, 1022.9387556204747} - {165.21409631559922, 1048.4693673490142} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - 0 - - - - - Bounds - {{84.918390063136314, 1048.4693603515625}, {108.00000616531918, 60.376010894775391}} - Class - ShapedGraphic - ID - 9 - Shape - Cloud - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 State\ -Transition\ -Notifications} - VerticalPad - 0 - - - - Class - Group - Graphics - - - Bounds - {{253.38389312817009, 1036.7868945785101}, {27.016406012875592, 38.542124503311257}} - Class - ShapedGraphic - ID - 93 - Magnets - - {0.15027599999999999, -0.32002000000000003} - {-0.5, -0.49964799999999998} - {-0.5, -0.25638699999999998} - {-0.10728, -0.148201} - {0.041826500000000003, 0.088786000000000004} - {-0.043045800000000002, 0.088786000000000004} - {0.22847700000000001, 0.5} - {0.5, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - - Rotation - 90 - Shape - Bezier - ShapeData - - UnitPoints - - {0.406219, 0.101163} - {0.39736100000000002, -0.042958700000000002} - {0.31166700000000003, -0.185311} - {0.149117, -0.29534500000000002} - {-0.030395499999999999, -0.41686299999999998} - {-0.261517, -0.50514099999999995} - {-0.49668800000000002, -0.49976700000000002} - {-0.496693, -0.49976500000000001} - {-0.062913899999999995, -0.36058899999999999} - {-0.062913899999999995, -0.36058899999999999} - {-0.062918699999999994, -0.36058899999999999} - {-0.5, -0.21609700000000001} - {-0.5, -0.21609600000000001} - {-0.35843000000000003, -0.22182399999999999} - {-0.217449, -0.204378} - {-0.10928400000000001, -0.12870000000000001} - {-0.017806099999999998, -0.064687300000000003} - {0.032062500000000001, 0.0174179} - {0.040309900000000003, 0.101163} - {0.040309900000000003, 0.101163} - {-0.044847499999999998, 0.101163} - {-0.044847499999999998, 0.101163} - {-0.044847499999999998, 0.101163} - {0.22758200000000001, 0.5} - {0.22758200000000001, 0.5} - {0.22758200000000001, 0.5} - {0.5, 0.101163} - {0.5, 0.101163} - {0.5, 0.101163} - {0.406219, 0.101163} - - - - - Bounds - {{234.04078487632972, 1028.8634930208395}, {26.999999999999996, 38.288223134554855}} - Class - ShapedGraphic - ID - 94 - Magnets - - {0.15027599999999999, -0.32002000000000003} - {-0.5, -0.49964799999999998} - {-0.5, -0.25638699999999998} - {-0.10728, -0.148201} - {0.041826500000000003, 0.088786000000000004} - {-0.043045800000000002, 0.088786000000000004} - {0.22847700000000001, 0.5} - {0.5, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - - Rotation - 180 - Shape - 33C70F48-B008-4466-BD81-E84D73C055CA-438-0000056AF6035FFB - - - Bounds - {{242.15904544605092, 1007.9418767503257}, {27.016406012875589, 38.264972185430459}} - Class - ShapedGraphic - ID - 95 - Magnets - - {0.15027599999999999, -0.32002000000000003} - {-0.5, -0.49964799999999998} - {-0.5, -0.25638699999999998} - {-0.10728, -0.148201} - {0.041826500000000003, 0.088786000000000004} - {-0.043045800000000002, 0.088786000000000004} - {0.22847700000000001, 0.5} - {0.5, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - - Rotation - 270 - Shape - 33C70F48-B008-4466-BD81-E84D73C055CA-438-0000056AF6035FFB - - - Bounds - {{261.04078487632967, 1015.981498100793}, {27.000000000000004, 38.288223134554862}} - Class - ShapedGraphic - ID - 96 - Magnets - - {0.15027599999999999, -0.32002000000000003} - {-0.5, -0.49964799999999998} - {-0.5, -0.25638699999999998} - {-0.10728, -0.148201} - {0.041826500000000003, 0.088786000000000004} - {-0.043045800000000002, 0.088786000000000004} - {0.22847700000000001, 0.5} - {0.5, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - {0.40652500000000003, 0.088786000000000004} - - Shape - 33C70F48-B008-4466-BD81-E84D73C055CA-438-0000056AF6035FFB - - - ID - 92 - - - Bounds - {{396.52035685550777, 974.46163584936892}, {142, 14}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica-Bold - Size - 12 - - ID - 457 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\b\fs24 \cf0 ActionEngine (one impl.)} - VerticalPad - 0 - - Wrap - NO - - - Bounds - {{390.8163413254731, 1038.2328676107024}, {63.714366912841797, 31.333333333333332}} - Class - ShapedGraphic - ID - 450 - Shape - Rectangle - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs20 \cf0 Runner} - VerticalPad - 0 - - - - Bounds - {{478.53062537152402, 1038.2328900119185}, {63.714366912841797, 31.333333333333332}} - Class - ShapedGraphic - ID - 449 - Shape - Rectangle - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs20 \cf0 Runtime} - VerticalPad - 0 - - - - Bounds - {{478.5306334878843, 994.19207080251738}, {63.714366912841797, 31.333333333333332}} - Class - ShapedGraphic - ID - 447 - Shape - Rectangle - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs20 \cf0 Executor} - VerticalPad - 0 - - - - Bounds - {{390.81631892425446, 994.19204840129885}, {63.714366912841797, 31.333333333333332}} - Class - ShapedGraphic - ID - 446 - Shape - Rectangle - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs20 \cf0 Analyzer} - VerticalPad - 0 - - - - Class - LineGraphic - Head - - ID - 444 - - ID - 445 - Points - - {304.30400417385465, 1005.6686926988394} - {365.81839492659333, 1029.0751702876107} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - 0 - - - Tail - - ID - 423 - - - - Class - LineGraphic - Head - - ID - 10 - - ID - 433 - Points - - {437.73468537749687, 869.13090571936129} - {473.25508692784757, 868.8206769098332} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - TailArrow - FilledArrow - - - - - Bounds - {{473.75506787377572, 840.81631016602194}, {63.714366912841797, 56}} - Class - ShapedGraphic - ID - 10 - Magnets - - {0, 1} - {0, -1} - {1, 0} - {-1, 0} - - Shape - Cylinder - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\fs20 \cf0 Persistence\ -Backend} - VerticalPad - 0 - - - - Class - LineGraphic - ID - 428 - OrthogonalBarAutomatic - - OrthogonalBarPoint - {0, 0} - OrthogonalBarPosition - -1 - Points - - {258.38771438598633, 947} - {308.12470245361328, 886} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 2 - TailArrow - FilledArrow - - - - - Class - TableGroup - Graphics - - - Bounds - {{310.93862753220276, 826.66148410306198}, {126, 14}} - Class - ShapedGraphic - FitText - Vertical - Flow - Resize - ID - 426 - Shape - Rectangle - Style - - fill - - GradientCenter - {-0.29411799999999999, -0.264706} - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\b\fs24 \cf0 Storage} - VerticalPad - 0 - - TextPlacement - 0 - - - Bounds - {{310.93862753220276, 840.66148410306198}, {126, 28}} - Class - ShapedGraphic - FitText - Vertical - Flow - Resize - ID - 43 - Shape - Rectangle - Style - - fill - - GradientCenter - {-0.29411799999999999, -0.264706} - - - Text - - Align - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 - flow_name\ -- flow_uuid} - VerticalPad - 0 - - TextPlacement - 0 - - - Bounds - {{310.93862753220276, 868.66148410306198}, {126, 56}} - Class - ShapedGraphic - FitText - Vertical - Flow - Resize - ID - 427 - Shape - Rectangle - Style - - fill - - GradientCenter - {-0.29411799999999999, -0.264706} - - - Text - - Align - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 - save()\ -- get()\ -- get_failures()\ -...} - VerticalPad - 0 - - TextPlacement - 0 - - - GridH - - 426 - 43 - 427 - - - ID - 425 - - - Bounds - {{207.28567728426299, 974.78645878243321}, {105.10203552246094, 36}} - Class - ShapedGraphic - ID - 421 - Shape - Cloud - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Compilation} - VerticalPad - 0 - - - - Bounds - {{240.83671598660843, 957.79548143397199}, {38, 14}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 422 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Engine} - VerticalPad - 0 - - Wrap - NO - - - Bounds - {{215.83669066280191, 952.49403624989395}, {88, 72.509323120117188}} - Class - ShapedGraphic - ID - 423 - Shape - Rectangle - - - Class - LineGraphic - ID - 418 - Points - - {175.01475125757293, 858.46545582024169} - {175.01475125757293, 1126.5224146928358} - - Style - - stroke - - HeadArrow - 0 - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - 0 - - - - - Bounds - {{56.053440093994141, 802.0387135699907}, {88, 44}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica-BoldOblique - Size - 18 - - ID - 414 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\b\fs36 \cf0 Activation\ -Phase} - VerticalPad - 0 - - Wrap - NO - - - Bounds - {{105.08388984446533, 1003.3409264674543}, {59, 28}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 413 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Results/\ -Exceptions} - VerticalPad - 0 - - Wrap - NO - - - Class - LineGraphic - ID - 412 - Points - - {109.26527080670799, 991.99397346428293} - {192.17343756180355, 991.99397346428293} - - Style - - stroke - - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - FilledArrow - - - - - Bounds - {{115.18593484369178, 915.30610463461369}, {49, 56}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 411 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Run/\ -Resume/\ -Revert/\ -Suspend} - VerticalPad - 0 - - Wrap - NO - - - Class - LineGraphic - ID - 410 - Points - - {113.34690338032949, 979.62663303122656} - {203.34690321589053, 979.62663303122656} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - TailArrow - 0 - - - - - Class - Group - Graphics - - - Bounds - {{59.15303234416389, 922.95317062424022}, {35, 28}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 402 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 GillSans;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Library\ -User} - VerticalPad - 0 - - Wrap - NO - - - Class - Group - Graphics - - - Bounds - {{62.479807418169017, 1006.9767931370576}, {28.346457481384277, 28.346458435058594}} - Class - ShapedGraphic - ID - 404 - Shape - Rectangle - Style - - fill - - Color - - b - 0.4 - g - 1 - r - 1 - - Draws - NO - FillType - 2 - GradientAngle - 90 - GradientColor - - b - 0.4 - g - 1 - r - 1 - - MiddleColor - - b - 0.4 - g - 1 - r - 1 - - TrippleBlend - YES - - shadow - - Beneath - YES - Draws - NO - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - Draws - NO - Width - 1.5 - - - VFlip - YES - Wrap - NO - - - Class - LineGraphic - ID - 405 - Points - - {62.479807418169017, 981.46498235748606} - {90.826264899553294, 981.46498235748606} - {90.826264899553294, 981.46498235748606} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 406 - Points - - {76.653036158861156, 992.80356535003978} - {62.479806941331859, 1006.9767940907319} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 407 - Points - - {76.653036158861156, 992.80356535003978} - {90.826264899553294, 1007.5437581354097} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 408 - Points - - {76.653036158861156, 972.96104511307078} - {76.653036158861156, 992.80356773422557} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Bounds - {{68.149098914445872, 955.95317062424022}, {17.00787353515625, 17.00787353515625}} - Class - ShapedGraphic - ID - 409 - Shape - Circle - Style - - fill - - Color - - b - 0.4 - g - 1 - r - 1 - - Draws - NO - FillType - 2 - GradientAngle - 90 - GradientColor - - b - 0.4 - g - 1 - r - 1 - - MiddleColor - - b - 0.4 - g - 1 - r - 1 - - TrippleBlend - YES - - shadow - - Beneath - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - Width - 1.5 - - - - - ID - 403 - - - ID - 401 - - - Class - LineGraphic - ID - 399 - Points - - {450.39306747989752, 692.4796011495929} - {451.28062907089827, 609.48823926071918} - - Style - - stroke - - HeadArrow - UMLInheritance - Legacy - - LineType - 1 - TailArrow - 0 - - - Tail - - ID - 398 - Info - 2 - - - - Bounds - {{405.3877204726607, 692.97957255114432}, {90, 36}} - Class - ShapedGraphic - ID - 398 - Magnets - - {0, 1} - {0, -1} - {1, 0} - {-1, 0} - - Shape - Rectangle - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\b\fs24 \cf0 Distributed\ -Engine} - VerticalPad - 0 - - - - Class - LineGraphic - Head - - ID - 395 - - ID - 397 - Points - - {515.69384736152347, 643.69388126880108} - {479.18127560660326, 607.7427600751555} - - Style - - stroke - - HeadArrow - UMLInheritance - Legacy - - LineType - 1 - TailArrow - 0 - - - Tail - - ID - 396 - Info - 2 - - - - Bounds - {{470.69384736152347, 643.69388126880108}, {90, 36}} - Class - ShapedGraphic - ID - 396 - Magnets - - {0, 1} - {0, -1} - {1, 0} - {-1, 0} - - Shape - Rectangle - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\b\fs24 \cf0 No-Thread\ -Engine} - VerticalPad - 0 - - - - Class - LineGraphic - Head - - ID - 395 - - ID - 27 - Points - - {387.9411398922191, 643.33613420977974} - {422.69414909838434, 607.74967407906797} - - Style - - stroke - - HeadArrow - UMLInheritance - Legacy - - LineType - 1 - TailArrow - 0 - - - Tail - - ID - 11 - - - - Bounds - {{342.59180028217145, 643.69385172515479}, {90, 36}} - Class - ShapedGraphic - ID - 11 - Magnets - - {0, 1} - {0, -1} - {1, 0} - {-1, 0} - - Shape - Rectangle - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\i\b\fs24 \cf0 K -\i0 -Threaded\ -Engine} - VerticalPad - 0 - - - - Class - TableGroup - Graphics - - - Bounds - {{405.38771609711887, 495.39195656369293}, {90, 14}} - Class - ShapedGraphic - FitText - Vertical - Flow - Resize - ID - 393 - Shape - Rectangle - Style - - fill - - GradientCenter - {-0.29411799999999999, -0.264706} - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\i\b\fs24 \cf0 Engine} - VerticalPad - 0 - - TextPlacement - 0 - - - Bounds - {{405.38771609711887, 509.39195656369293}, {90, 42}} - Class - ShapedGraphic - FitText - Vertical - Flow - Resize - ID - 394 - Shape - Rectangle - Style - - fill - - GradientCenter - {-0.29411799999999999, -0.264706} - - - Text - - Align - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 - notifier\ -- atom_notifier\ -- storage} - VerticalPad - 0 - - TextPlacement - 0 - - - Bounds - {{405.38771609711887, 551.39195656369293}, {90, 56}} - Class - ShapedGraphic - FitText - Vertical - Flow - Resize - ID - 395 - Shape - Rectangle - Style - - fill - - GradientCenter - {-0.29411799999999999, -0.264706} - - - Text - - Align - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 - compile()\ -- prepare()\ -- run()\ -- suspend()} - VerticalPad - 0 - - TextPlacement - 0 - - - GridH - - 393 - 394 - 395 - - - ID - 392 - - - Bounds - {{324.43479725203395, 600.47359077973181}, {35, 14}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica - Size - 12 - - ID - 390 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\fs24 \cf0 Load()} - VerticalPad - 0 - - Wrap - NO - - - Class - LineGraphic - ID - 389 - Points - - {299.19385094739675, 622.37332926589443} - {349.19384944761669, 622.37332926589443} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - TailArrow - FilledArrow - - - - - Class - LineGraphic - ID - 388 - Points - - {315.73465810741618, 475.31037359821562} - {315.73465810741618, 743.36733247080952} - - Style - - stroke - - HeadArrow - 0 - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - 0 - - - - - Bounds - {{183.17344081803969, 484.69386811754907}, {72, 42}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica - Size - 12 - - ID - 387 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\fs24 \cf0 Workflow +\ -Runtime\ -Configuration} - VerticalPad - 0 - - Wrap - NO - - - Bounds - {{192.17344567816366, 658.51018444452041}, {54, 36}} - Class - ShapedGraphic - ID - 386 - Shape - Rectangle - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Storage\ -Config} - - - - Bounds - {{192.17343756180355, 606.36734600615216}, {54, 36}} - Class - ShapedGraphic - ID - 385 - Shape - Rectangle - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Engine\ -Config} - - - - Bounds - {{192.1734294454435, 554.22448349078456}, {54, 36}} - Class - ShapedGraphic - ID - 1 - Shape - Rectangle - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Flow} - - - - Bounds - {{161.23465842199704, 531.90531247737556}, {126, 182}} - Class - ShapedGraphic - ID - 15 - Shape - NoteShape - Style - - Text - - VerticalPad - 0 - - - - Bounds - {{81.034519768316954, 649.04956274775338}, {43, 28}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 384 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Returns\ -Engine} - VerticalPad - 0 - - Wrap - NO - - - Class - LineGraphic - ID - 383 - Points - - {83.544734716513915, 635.76110575309315} - {129.28172189203261, 635.76110575309315} - - Style - - stroke - - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - FilledArrow - - - - - Bounds - {{80.054942197373691, 597.00601059533471}, {47, 14}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 382 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Provides} - VerticalPad - 0 - - Wrap - NO - - - Class - LineGraphic - ID - 381 - Points - - {87.626367290135391, 623.39376532003678} - {133.36335446565408, 623.39376532003678} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - TailArrow - 0 - - - - - Class - Group - Graphics - - - Bounds - {{33.432496253969788, 566.72030291305043}, {35, 28}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 373 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 GillSans;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Library\ -User} - VerticalPad - 0 - - Wrap - NO - - - Class - Group - Graphics - - - Bounds - {{36.759271327974915, 650.74392542586781}, {28.346457481384277, 28.346458435058594}} - Class - ShapedGraphic - ID - 375 - Shape - Rectangle - Style - - fill - - Color - - b - 0.4 - g - 1 - r - 1 - - Draws - NO - FillType - 2 - GradientAngle - 90 - GradientColor - - b - 0.4 - g - 1 - r - 1 - - MiddleColor - - b - 0.4 - g - 1 - r - 1 - - TrippleBlend - YES - - shadow - - Beneath - YES - Draws - NO - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - Draws - NO - Width - 1.5 - - - VFlip - YES - Wrap - NO - - - Class - LineGraphic - ID - 376 - Points - - {36.759271327974915, 625.23211464629628} - {65.105728809359192, 625.23211464629628} - {65.105728809359192, 625.23211464629628} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 377 - Points - - {50.932500068667053, 636.57069763884999} - {36.759270851137757, 650.74392637954213} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 378 - Points - - {50.932500068667053, 636.57069763884999} - {65.105728809359192, 651.31089042421991} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 379 - Points - - {50.932500068667053, 616.728177401881} - {50.932500068667053, 636.57070002303578} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Bounds - {{42.42856282425177, 599.72030291305043}, {17.00787353515625, 17.00787353515625}} - Class - ShapedGraphic - ID - 380 - Shape - Circle - Style - - fill - - Color - - b - 0.4 - g - 1 - r - 1 - - Draws - NO - FillType - 2 - GradientAngle - 90 - GradientColor - - b - 0.4 - g - 1 - r - 1 - - MiddleColor - - b - 0.4 - g - 1 - r - 1 - - TrippleBlend - YES - - shadow - - Beneath - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - Width - 1.5 - - - - - ID - 374 - - - ID - 372 - - - Bounds - {{56.053440093994141, 454.08162381538807}, {97, 44}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica-BoldOblique - Size - 18 - - ID - 371 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\b\fs36 \cf0 Translation\ -Phase} - VerticalPad - 0 - - Wrap - NO - - - Class - LineGraphic - ID - 370 - Points - - {239.72452365274654, 129.59183421248156} - {249.59182766764883, 184.82352424792543} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - 0 - - - - - Bounds - {{203.06122053766794, 96.734692512775737}, {75, 28}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica - Size - 12 - - ID - 369 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\fs24 \cf0 Explicit\ -dependencies} - VerticalPad - 0 - - Wrap - NO - - - Class - LineGraphic - ID - 368 - Points - - {16.938771677235714, 434.69386909068618} - {550.61223067824244, 434.69386909068618} - - Style - - stroke - - HeadArrow - 0 - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - 0 - - - - - Bounds - {{56.053440093994141, 39.83673387962002}, {112, 44}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica-BoldOblique - Size - 18 - - ID - 367 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\b\fs36 \cf0 Construction\ -Phase} - VerticalPad - 0 - - Wrap - NO - - - Bounds - {{100.22448568729375, 191.89795101130832}, {50, 56}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 366 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Creates\ -\ -\ -Workflow} - VerticalPad - 0 - - Wrap - NO - - - Class - LineGraphic - ID - 364 - Points - - {110.31631892346081, 220.32652202282102} - {156.05330609897936, 220.32652202282102} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - TailArrow - FilledArrow - - - - - Class - TableGroup - Graphics - - - Bounds - {{306.40872322346235, 337.6122433290239}, {126, 14}} - Class - ShapedGraphic - FitText - Vertical - Flow - Resize - ID - 361 - Shape - Rectangle - Style - - fill - - GradientCenter - {-0.29411799999999999, -0.264706} - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\i\b\fs24 \cf0 Retry (Atom)} - VerticalPad - 0 - - TextPlacement - 0 - - - Bounds - {{306.40872322346235, 351.6122433290239}, {126, 56}} - Class - ShapedGraphic - FitText - Vertical - Flow - Resize - ID - 362 - Shape - Rectangle - Style - - fill - - GradientCenter - {-0.29411799999999999, -0.264706} - - - Text - - Align - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 - execute()\ -- revert()\ -- on_failure()\ -...} - VerticalPad - 0 - - TextPlacement - 0 - - - GridH - - 361 - 362 - - - ID - 360 - - - Class - TableGroup - Graphics - - - Bounds - {{165.22448860432195, 337.6122433290239}, {126, 14}} - Class - ShapedGraphic - FitText - Vertical - Flow - Resize - ID - 42 - Shape - Rectangle - Style - - fill - - GradientCenter - {-0.29411799999999999, -0.264706} - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\i\b\fs24 \cf0 Task (Atom)} - VerticalPad - 0 - - TextPlacement - 0 - - - Bounds - {{165.22448860432195, 351.6122433290239}, {126, 56}} - Class - ShapedGraphic - FitText - Vertical - Flow - Resize - ID - 44 - Shape - Rectangle - Style - - fill - - GradientCenter - {-0.29411799999999999, -0.264706} - - - Text - - Align - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 - execute()\ -- revert()\ -- update_progress()\ -...} - VerticalPad - 0 - - TextPlacement - 0 - - - GridH - - 42 - 44 - - - ID - 352 - - - Class - LineGraphic - Head - - ID - 840 - - ID - 842 - Points - - {381.22447887295158, 117.34693649161716} - {381.85914273540726, 165.11464199273692} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - 0 - - - - - Class - LineGraphic - ID - 347 - Points - - {395.51019288062668, 111.79728682683641} - {394.92858042750709, 139.79728698730469} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - 0 - - - - - Bounds - {{302.34693227416039, 79.112244302955403}, {185, 28}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica - Size - 12 - - ID - 345 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\fs24 \cf0 Workflow (declarative) structure\ -& code (not executed immediately)} - VerticalPad - 0 - - Wrap - NO - - - Class - LineGraphic - ID - 329 - Points - - {472.22448860432172, 249.61224332902393} - {411.31888545619614, 269.91411104506579} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - 0 - - - - - Class - LineGraphic - ID - 343 - Points - - {474.22448860432172, 205.97599760148498} - {409.22448860432172, 228.24848490451151} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - Pattern - 1 - TailArrow - 0 - - - - - Bounds - {{478.22448860432172, 179.61224332902398}, {83, 42}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica - Size - 12 - - ID - 344 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\fs24 \cf0 Dataflow\ -(symbol-based)\ -dependencies} - VerticalPad - 0 - - Wrap - NO - - - Class - LineGraphic - ID - 341 - Points - - {361.09295431543518, 212.90662923905811} - {315.35596770538763, 253.81785993690883} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - TailArrow - 0 - - - - - Bounds - {{387.44897720864344, 229.36224332902393}, {30, 14}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 336 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 out/in} - VerticalPad - 0 - - Wrap - NO - - - Class - LineGraphic - ID - 334 - Points - - {239.72451559473222, 278.11224001641114} - {269.72448860432183, 278.11224001641114} - - Style - - stroke - - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - FilledArrow - - - - - Class - LineGraphic - ID - 333 - Points - - {324.22448507715393, 278.11224001641114} - {354.2244580867436, 278.11224001641114} - - Style - - stroke - - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - FilledArrow - - - - - Class - LineGraphic - ID - 332 - Points - - {325.22451559473205, 192.11224001641122} - {355.22448860432172, 192.11224001641122} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - TailArrow - 0 - - - - - Class - LineGraphic - Head - - ID - 324 - - ID - 331 - Points - - {239.72450209952797, 192.61225527520028} - {269.72447510911758, 192.61225527520028} - - Style - - stroke - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - TailArrow - 0 - - - Tail - - ID - 28 - - - - Bounds - {{474.22448860432172, 234.61224332902393}, {49, 42}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Font - Helvetica - Size - 12 - - ID - 330 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\i\fs24 \cf0 Nested\ -subflow\ -with retry} - VerticalPad - 0 - - Wrap - NO - - - Bounds - {{271.72448860432172, 251.61224332902393}, {54, 54}} - Class - ShapedGraphic - HFlip - YES - ID - 328 - Shape - Circle - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Task} - VerticalPad - 0 - - - - Bounds - {{183.72451554562139, 251.61224332902393}, {54, 54}} - Class - ShapedGraphic - HFlip - YES - ID - 327 - Shape - Circle - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Task} - VerticalPad - 0 - - - - Bounds - {{355.22448860432172, 165.61224332902398}, {54, 54}} - Class - ShapedGraphic - HFlip - YES - ID - 840 - Shape - Circle - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Flow} - VerticalPad - 0 - - - - Bounds - {{270.22448860432183, 165.61224332902398}, {54, 54}} - Class - ShapedGraphic - HFlip - YES - ID - 324 - Shape - Circle - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Task} - VerticalPad - 0 - - - - Bounds - {{185.22448860432195, 165.61224332902398}, {54, 54}} - Class - ShapedGraphic - HFlip - YES - ID - 28 - Shape - Circle - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Task} - VerticalPad - 0 - - - - Class - TableGroup - Graphics - - - Bounds - {{165.224488604322, 153.79728666398492}, {269, 168}} - Class - ShapedGraphic - FitText - Vertical - Flow - Resize - ID - 35 - Shape - Rectangle - Style - - fill - - GradientCenter - {-0.29411799999999999, -0.264706} - - - Text - - Align - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 \ -\ -\ -\ -\ -\ -\ -\ -\ -\ -\ -} - VerticalPad - 0 - - TextPlacement - 0 - - - Bounds - {{165.224488604322, 139.79728666398492}, {269, 14}} - Class - ShapedGraphic - FitText - Vertical - Flow - Resize - ID - 34 - Shape - Rectangle - Style - - fill - - GradientCenter - {-0.29411799999999999, -0.264706} - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\i\b\fs24 \cf0 Flow (pattern)} - VerticalPad - 0 - - TextPlacement - 0 - - - GridH - - 34 - 35 - - - ID - 33 - - - Class - Group - Graphics - - - Bounds - {{56.122447887295152, 164.67346775924003}, {35, 28}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - ID - 61 - Shape - Rectangle - Style - - fill - - Draws - NO - - shadow - - Draws - NO - - stroke - - Draws - NO - - - Text - - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 GillSans;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 Library\ -User} - VerticalPad - 0 - - Wrap - NO - - - Class - Group - Graphics - - - Bounds - {{59.449222961300279, 248.69709027205738}, {28.346457481384277, 28.346458435058594}} - Class - ShapedGraphic - ID - 63 - Shape - Rectangle - Style - - fill - - Color - - b - 0.4 - g - 1 - r - 1 - - Draws - NO - FillType - 2 - GradientAngle - 90 - GradientColor - - b - 0.4 - g - 1 - r - 1 - - MiddleColor - - b - 0.4 - g - 1 - r - 1 - - TrippleBlend - YES - - shadow - - Beneath - YES - Draws - NO - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - Draws - NO - Width - 1.5 - - - VFlip - YES - Wrap - NO - - - Class - LineGraphic - ID - 64 - Points - - {59.449222961300279, 223.18527949248588} - {87.795680442684557, 223.18527949248588} - {87.795680442684557, 223.18527949248588} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 65 - Points - - {73.622451701992418, 234.52386248503961} - {59.449222484463121, 248.6970912257317} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 66 - Points - - {73.622451701992418, 234.52386248503953} - {87.795680442684557, 249.26405527040947} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Class - LineGraphic - ID - 67 - Points - - {73.622451701992418, 214.68134224807059} - {73.622451701992418, 234.52386486922538} - - Style - - shadow - - Beneath - YES - Draws - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - HeadArrow - 0 - Legacy - - LineType - 1 - TailArrow - 0 - Width - 1.5 - - - - - Bounds - {{65.118514457577135, 197.67346775924003}, {17.00787353515625, 17.00787353515625}} - Class - ShapedGraphic - ID - 68 - Shape - Circle - Style - - fill - - Color - - b - 0.4 - g - 1 - r - 1 - - Draws - NO - FillType - 2 - GradientAngle - 90 - GradientColor - - b - 0.4 - g - 1 - r - 1 - - MiddleColor - - b - 0.4 - g - 1 - r - 1 - - TrippleBlend - YES - - shadow - - Beneath - YES - Fuzziness - 2.5038185119628906 - ShadowVector - {0, 1} - - stroke - - CornerRadius - 1 - Width - 1.5 - - - - - ID - 62 - - - ID - 60 - - - Class - Group - Graphics - - - Bounds - {{524.20410965184897, 903.84686831164879}, {90, 36}} - Class - ShapedGraphic - ID - 440 - Magnets - - {0, 1} - {0, -1} - {1, 0} - {-1, 0} - - Shape - RoundRect - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\fs20 \cf0 Zookeeper} - VerticalPad - 0 - - - - Bounds - {{524.20413205306681, 867.84684591043094}, {90, 36}} - Class - ShapedGraphic - ID - 441 - Magnets - - {0, 1} - {0, -1} - {1, 0} - {-1, 0} - - Shape - RoundRect - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\fs20 \cf0 Filesystem} - VerticalPad - 0 - - - - Bounds - {{524.20410965184885, 832.86723361478039}, {90, 36}} - Class - ShapedGraphic - ID - 442 - Magnets - - {0, 1} - {0, -1} - {1, 0} - {-1, 0} - - Shape - RoundRect - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\fs20 \cf0 Memory} - VerticalPad - 0 - - - - Bounds - {{524.20409341912841, 797.78565525800093}, {90, 36}} - Class - ShapedGraphic - ID - 443 - Magnets - - {0, 1} - {0, -1} - {1, 0} - {-1, 0} - - Shape - RoundRect - Style - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\fs20 \cf0 SQLAlchemy} - VerticalPad - 0 - - - - ID - 439 - - - Bounds - {{366.28570556640625, 1120.9999961853027}, {75.71429443359375, 44}} - Class - ShapedGraphic - ID - 835 - Shape - Rectangle - Style - - stroke - - Pattern - 1 - - - - - Bounds - {{366.28571101041302, 969.38000187999}, {202.04080200195312, 196.6199951171875}} - Class - ShapedGraphic - ID - 444 - Shape - Rectangle - - - Bounds - {{379.54083251953125, 960.24970708018532}, {202.04080200195312, 196.6199951171875}} - Class - ShapedGraphic - ID - 1170 - Shape - Rectangle - - - Bounds - {{181.81308267749165, 1321.1440843224241}, {236.99999999999997, 168}} - Class - ShapedGraphic - FitText - Vertical - Flow - Resize - ID - 1172 - Shape - Rectangle - Style - - fill - - GradientCenter - {-0.29411799999999999, -0.264706} - - - Text - - Align - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf200 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 \ -\ -\ -\ -\ -\ -\ -\ -\ -\ -\ -} - VerticalPad - 0 - - TextPlacement - 0 - - - GridInfo - - GuidesLocked - NO - GuidesVisible - YES - HPages - 2 - ImageCounter - 1 - KeepToScale - - Layers - - - Lock - NO - Name - Layer 1 - Print - YES - View - YES - - - LayoutInfo - - Animate - NO - circoMinDist - 18 - circoSeparation - 0.0 - layoutEngine - dot - neatoSeparation - 0.0 - twopiSeparation - 0.0 - - LinksVisible - NO - MagnetsVisible - NO - MasterSheets - - ModificationDate - 2014-07-09 22:24:00 +0000 - Modifier - Joshua Harlow - NotesVisible - NO - Orientation - 2 - OriginVisible - NO - PageBreaks - YES - PrintInfo - - NSBottomMargin - - float - 41 - - NSHorizonalPagination - - coded - BAtzdHJlYW10eXBlZIHoA4QBQISEhAhOU051bWJlcgCEhAdOU1ZhbHVlAISECE5TT2JqZWN0AIWEASqEhAFxlwCG - - NSLeftMargin - - float - 18 - - NSPaperSize - - size - {612, 792} - - NSPrintReverseOrientation - - int - 0 - - NSRightMargin - - float - 18 - - NSTopMargin - - float - 18 - - - PrintOnePage - - ReadOnly - NO - RowAlign - 1 - RowSpacing - 36 - SheetTitle - Canvas 1 - SmartAlignmentGuidesActive - YES - SmartDistanceGuidesActive - YES - UniqueID - 1 - UseEntirePage - - VPages - 3 - WindowInfo - - CurrentSheet - 0 - ExpandedCanvases - - - name - Canvas 1 - - - Frame - {{77, 45}, {1067, 833}} - ListView - - OutlineWidth - 142 - RightSidebar - - ShowRuler - - Sidebar - - SidebarWidth - 120 - VisibleRegion - {{8.8235295767602651, 949.50982167692416}, {900.00001682954701, 665.68628695780228}} - Zoom - 1.0199999809265137 - ZoomValues - - - Canvas 1 - 1.0199999809265137 - 1 - - - - - diff --git a/doc/diagrams/core.graffle.tgz b/doc/diagrams/core.graffle.tgz new file mode 100644 index 0000000000000000000000000000000000000000..9ab233217defca044b593a26f00d7ed13c36ef70 GIT binary patch literal 16344 zcmV;}KPSK+iwFQR&~H=#1MFQ}bKADI?rZ!jbnRAz`=w2@CvlQCowUj1($3!F z7h0knHnQkV$xYKt|NB`Wbs<53q-0sX1jx5zhM8c3BLVFw#`2p1x~c(HUl-F^oDT-ye2DXL_UZT7 z-!93gyBN@JUl0HF=h=Ah2RvJvO}@X6BXoa|OorJgot%Du|NXn?`|AF`9_>B+pQo=5 z-~Idbv%&c|oz4cY-#>Zz;&5<(|KQ;9`T00GIC%Q*>EQLt7jNGUppS!tXTRJZ-2X6} zoqv09@afa1C^oI4VRmL3P7hw^*?E%BKEH&{_JJT8%|`d(Hf8U%;lQ_}bU1sockkiH z*(^(FW{0K%EWoMK0G|w*1qgU|hhf+9T#19YVjVkRi)e%EhaQ9ACbm2Ltr? zgTWV!Ie9Q3SnI$4u8r7G9FF5@llW~mPFHueNM1Z$wO>pusm^iP?CtFHI9at^jJdox z{Sc3`Ps^*7%jNZ_dHiWLtkuANc~u<_OsUoFEYChBTkLlEY4xF8nrZT%EIaGLYlrdV zk9hhjPfybomo3()SfVP66cpL`;%xGGoSse^{o`i;MeDcc@esOLw$GDz{D_K&2gQd) zQ~1O1{L^@r*h3)z!23w};Ne1a;@Vx;Fcn!=%)jN3h=XsXT-hPPBlc_^g>rC)sGCfa*FvcH@ z$X1x4mfUZKY6eS}+MwE#8KEoC9ENPNcdMMFzruLs8 zr1lvTV*klgh4-H+_EgeC#QEXleMZ%Oq5imd{9GW)pFe-%x=VI39+k7vywSg(y>*Zj zBJnz&eejD#GEUBt$*d+B%h|2BuTxYAR>S!D)vwRq)dgxncAljkqAw_7NN~c@@qx#h~#?pDn>+MG?f7%y4C44Ge;Ddc8&YOR}@zkmuxb(W<~`U zsKO7p=$#{uHCo-0?`)pU&C3VWS}?g9f?9K8+s5|t#V?l`8sUN}vH8$C*PL>t=Ym57 zA$lXhF~q3te-8%eJr0dQ@*vv8tDI1BxZ0OmSEW zp#b!;fH@*sMp7y+3d4!mBHd2R1&Z0EI8WcEVsYM_OaBVwD0_iM0_Iip&+4!|7q&!Vh(YZc3`> z>CDJ!Pv7zRxE7W|k4r6P!3)U=dwIV4z zcTB1-yoEzP`IHX%EN08~gu_`pIUPG5vO`o((($-WQ0xGdF9KCZ-9iYhh}l;A?iBD% z`gSP@|IEFPN7Vs118VM`R)Teu&ra}BJej7&f5Y=rKu&Tr%!XNPE`jq{{xQ8chQGlY z-JAbC%##FI7r+{SIWq6F6569z+1m2<5(RrMYj%I&?-wOX@Oo}(5%o`!BcvHx{8MEEI>7of^MH7yS zCO~c81TLBg2~AADX+DW(7kNBB`tNXW&x|;QF%vd88lIrRv$J$IwKa#CHc+SL8OHr4 z-CoGw09~3wQ|AOPDsEHdij7sSB;a`=B$+mW1E#5gSwSNSIuuN5Du~o}XkW9}AxIW3ujrV@ZX6ea1 znYdM~-3fx?jS*Btnkt2<#DpqIlr&BYW|0731jj{FB!ultE~z<3k-&sVA~B_!YSB#{ zyq2BdriZ)cxip)v7T&gpTkILPLAmA>Nf`kigJQ*$Gp!IeJi%E6dLFn;BZ;{xoFe7Q z!nudv2a0{Jk>5FTx<{=Jl0yvZf=HdqY=M54hvF8Ddye>$<)>IpJT|}I>aY70K9A*U z?qZAimhV#k!=WA>vs;Jy7H`JWV^FzukLDOgO+~48|B;;=RU420>CRrb)ot4vG*`Ip z;P_dk=18xDbzBrsr?k*YFfc6({YSA0`3bD4WF`$%7Vp3%A#NksHRC}pS+Qb5tB#j^ zx&I^^k6s;*)Bj%B_i@IE?k%E3suCq~EwdkuSwQ1}#sQ7H(fD;Xox#)${x?25|88)! zcL+|<#aV&SM|-b7#M9(Pp>)HC@cBhnJ>&X82AI!CnG$A2l}6y(ltBj;X#su?LzFTw zat*?pawU;nkp=Y1i-YCoNMx6Kvl#U!Gv)@YFJP%w{3xYjZe;<#^=xR z=_Hxe?^UhQ-9qW-CRL4_lpC&9VAm`nTfc;*S>46esySJ=lfGqF8`MK>pXSC zvV+kuGI{#S5K9RmRT=AEbSxF9{R(%N!YY_hvhqaj5|3zMP1MFx^v$qrduW3SAa%M@ z+D?0d%!Y&o5P=gaS?!T;D5YT`$(3C*-i9N)BH=wBVB76FHIb!uYo)uJ#;Or^MKT1~ zc^do|a9%W0vM{7Yq-!rx7}CbvCKQ7*;!JVAJ40FoLpm7JUzH(^)jF2+b~dJ723d5Z z88P`Uu#G7->K`+~BMRzYFhUsxgmJm(N|(qYn3e!Yyx>H=J7kdoSpu?rRmei*twI)} z95#!j5tY)|APVdkf)~hAd$Fd?BDP*@3?9YmHLLPg&sFK7Ldyw{e(pvde} zP>6s{#F@tBHZg@F0X~$oPscWr> z_Eh17s+}#QjO{%O`-4w$I{U{zkM<&X-Kf%>^`MbZFlhh;%^oa_gp}F`jRJHQyIaNL z+`T6tKx2T$0F9eNV|*M>M%l!mCHe2!@znfvi*eFysu2oPlZ%DfCB^X&PMl=y z?%^Z}I2mv<;N+HZ@;sZ)N{}3m4nJkx73=y;4l`|9pYbk<&pBNUt{nJG02sjl>LV$!*!?PUINHVFg5KIC? zS`pqIZ=Zhu^UD{L4@sT^&kda{cSU=tFTFE*)s260YC>6M2m6qrOPy1}cn=N%VA2SH z$lfA&i5(<9UU!|K)W}RYncbXX<>O_zw(q@&tpl>F*Y=win@qly-5=rPjJ52C`c|=p z;qadf@FY1;@}s?rqdZ~=D}pJLR0+%r)sKojTEgVM;H{`X;OfDE zZ&AX&-aT5(b!Jo8qt#R8Ui}^|+skU-r4@E`ZNaxnYu?qN@u>#CXUE3DdYQ$e#pic9 zOwTx;CFYOfM^E7M*Rq9lW_uk;1T{qG6V4BWiRcCOjBW-jBIECN^lH%w$!Xm*K zBiNLp!L=eKYmSs_#?ySy(nocy0K6Gh^Q({*l@(cQRV7QRy8l_ja@;6`bK`maR&FoU zjqA-N@w#t@>C zn=|UDER!FaFlpAQ*tuYSAWOQy+W%;M0lN*d+daO{3L3U^J`4k!9bPzZxN|Iw&^HrojV$O$IP@lF^<5pVdKs#xh8e!3&2j;tg zenD=|`nHoK%LWUUXp}J1-4?H>>o%*c1y!CTlce^;mun1TJED2<*I(%*aiRZ0rbxsQRakMb$OR}P zB>Xb=wwQrmiyt>_8G);+y$)3jxJFF85Cr6bSUl-Y%dyJo^Me%DupmGpd^Hq4st zrT1Dy8`%8rfpr)JzCwgrA6@2v2k@N*3d@TP`>MiIZi@+#NK2+L0=vna^5EmTWiNu8 zp9lrF91}&&uf6ZOm-^Y%^dcyyTMlO%rwcF0A`17iR4I4fH0hp~Sc zpw>gPsAYM%THI2naTK{M8~7`ys$DpXUe+4;6@u}Ng2hJBT!nTsSmZo1EZT z`az5$p_!yqP~%N@1KJW?xuozPsd=N4o5R5!XJ3`m8}KaPS-`V;4_^zOHBPFp&|V#$ zIgT~6dT|rVF@u}6Yw;;6gTEaK)I zI|LpV1$U`9bVT0yO%kNZ3tnq0Y?26@B*G?%zBRLl*<^HK)MVHv(Jla;`y>!IhuW?x zI}9o_#v*fU6u7>KNG&f}3hQ@q*V;zWs>-Rm;@RukUiX#;j}zc?m|YG)r}^(S{yWTf zui@WL=PSk9XeUFL@7m?L_VHU=yw*NG>s?%{RuA{PBKq1MtULqJD+=vylM=03{DcBMf;;UD%>lPQ1 z=E02{7Z(!y1m9awT*#U0P*3_*78lZR0`t|}FBBJs;=)i|xQ*gMCZsEmPj6ju;VnBa z`71KGgEx44)+%^|x6B(1#f96Licnm5>9iE{SNE@-mU0^|oTm1%mCv)p;ixtvL3P)ZX({+jqS3p=jcanzU!WC`6p9*onpIeSUTP$wwg+Ch^ zKmb6YxNu1u0H{J70O*zj&=#wk-2jBpI>m+UAQp-X0{~gX0f4T(xR6QQEH0!<-Lm3B z=C893#f47FkWLEeq%H#WKy5hh>ZCO zgZfK67&yMPg{JaQRa9sQMD;Awx;U(QSn;Y$61!p#OTVl_QbC%Agf2& zK#4`>3SU$M7prgsLy;m3GetqQVKNUlu;Ms*fi)Kz5j$kDq^3$nL@JO*f)O?A*iGG7 z8w*P8nQExN22wjc;Kdk?1)MA{H*u`#|JFQTug~lEcKv>C=a6a~yktcmQ`!n@C2pEV zgb5~uM9QShipoU>T8A^jO*ESbO0|(S4#o(xOVZm|-miM*{)SbC?z6$l*0u8UmeO-S z?*k4vSl`lT?WK=>n9%Y}8!Xsx(R|KX&E#;}y4&o#Q*<`z+r}ho!L2dWm6BJRxa@&o z4b&^a=e7WFaRS57oa9$V# zz$0Q_T!|#noM;m}koFJ1t*8Zy5LoFZUci{(l+3Alu&=MgzV@(8bJx-$)@|)rs`0=& z(Isx&WyVab5zKdLxP)?SVoR|0f(yz-p(iPSFd%B%u_MP$Qhy-Tp)p%^rbDu-vZWfo z2G$;)^6h162~o{IdlF8L4Q!6=OYiM)PF;F6cWDNRw|W+L!5Z9_OEr_TEpE=3daP#RKi_7`@CpyOSHSKcCZxNmw7M6~SD|!tY>| zN1%gEIkC{7S&LXbQ4v5Zd^kg|(bCsS9XB&-sFVot4>D!a;?J4)}gu6H0-kof}3#-wRTE7dA*Oo|c z&17HlfC^9gyvLTkO~u2PaqpdP@65J$<=S_B#zsqa-J4tQaW02ruVve^ITiaFc4l(` zEjq8QF1JhLIZifkdfCMV`%H+%(!2FO(vhZEVqh`NbtAJ?>p|g2VJQS=f^tR?V!P_W zStZd^YhSzMx6s{w(QhlS@fFwjgDBMjyXsy&A(Y~hU7+g?r4$!ii*x-Um0knfBCcH% z;%L{&IBO%0kda{ERS;=*Fd`+y{A^(&5z!{{XK}=x)ZS$zF=`j{WtU z35X&H6ey4auPRNnMB>Wmi| zFYZTXyr{na&EQ31$EPp6&_>BtWdMWD@c%8ti{*&p7)gL>N5+ec7a1>7pH}ugkm3c3 zW>!9`mn24d5fYl6}`#2{ssnD<*r46J{(ZO;QGQllU8JhbEjX17CjI~?e2WVgef zxgDOc(%zJ2>z+kNaY=>#j%hbqn{H`0?Xkobf>Cth>xy(Q}efIPJjhZ3Fy(|!GsL7)CVSzO9VEqM>1#xr4 zs#OlVnVjT(H;ePw*jZ@o`E_S!A)R{Fot=41u(QH+>c-A2B-oiGyk&M)8FoDEEHwNL z{jf8Zqb56xVb{thu{JY1tIk|jirZpmI>FBNV1YF8AhWaG3u6E5Ea)f7&YJ1Y-Pu{X zl3#ar=3JAVl_ps?c4lyqoy|`VeY8|-Wk7D(E8liAtPu`_&{>@0RDlhL|3(i_X&*%|r-J1b1GZtP6E1UuvLnVnUK z9S=K;&CS|g*cr=Flbw0mwem@<&CJfKGnbX(cG#IAn_y>qut3tzo6OFRj-ADB?5sV+ z(S$?^c%~So2s2mkmO@z|3z(Hue?-`Qy#nDfGPrJ~-dRbr#E>4!^~Typb84w{h^c&J zx!(Is3jQ~kB;n>vE6cRdgnAoGK1E3aK1_xbhkQ&o%Q0=W;a{Q5$_I7$7l|Dw1p zeLGr)_l6E&I-*0pT;2hKL#p2I69$L8+(psPoAi8GRPP`7X|MKno%dD-J0>$;y0x5P zL@}T>g#gT(jfC|$uL1=dz4yQ?F4ng;=7TjUo;W0QF=*$Cjgb(PCNd6z=hFIm4*Zj- zh@G%KA5%kHcR-6Zw25>#uFpNh#knu6B{YfJ)|*fsJ@kNh!;tj@16aF6$r%aEyvMTy zT$~qwYCfb)^0Sp3zfQfOqVFePD7T{MeC^zdV%OdB-VSc2gR6@Zeq3orajdNzKrp80 z_{NjroZ@;eyib;i*&e>L@Ejk3-kd4D$j6+Bf$ZQ;DP^Hun}xQXPO_T~7?*XzNw5qPz2+c7EOEYnIHNRbSXAk!X6_~ts=X9@3B z!p}c`9$ejC?_9zmFkD-&Ju6)#^p*+?2gL@UcFubg=`SSQJUewg7OC5MYxM|w5sr~f zmbN#yTI^Uha2|z8^*yY!)5-1V+wU$;&Q5P`Z@vyjSMotAwRip4mw*{*6^#cpz(zzU zs=jDZ;YRjj9_w_)iqnof#BI`bXElruPN(qCZlCQ zWz>0_v!89weyTHP&affQkXb$k6dc>A~0v;X)s{jbw^@87=VPhxZ3{d#xc{Qb+9ug}lr!1?=g8T9&9bIhCPKm7Rq z-Rs$b*Z+9^?$_dwIw+hDq_n}{#ABc>pfsY?h=uRoLtqn4hXSWTZz(J2=wW(uj*tw~ znbhtYc$1{|^_SuFX3s#eKfk?AIet3%?WIc%*@)h0!|&JF0)EAb3z0F|Ta5u*{r1K? zWesYnDdU)E+rmMHQK2XB0T|H@+9w##eHb46h^2`niq-)C!|L?uS@TG-k#sEM19{as zFg)tnsn-gE*L;XD1P5d`-79N_XxQuV?N#0@-9a#Hv3qcF^6Nz2g@k@yoV*=Rhrjd9 z%cov3csB*bmX_Repg{87!Psbm)>K5IdKIGQW#U3~MwdD7foZR}S2dr>NJ|Wz5S2q~ zWQ!Mo=gie}F4B+JWmP5CRdr>h>m~nm`DrxhRCP=%)*d#vdICAHwA)JbaI?pBH3{7n zW01&WB^AA%O_UmAormbVjxjpeSd23mt#0;dtUMR%L@8obc?1c7)-m#oR>Pu~{y~kP z(N#jJ?Wz~yN3;^jJU`jUCET(RZuLLH^62bi9(I?PNED%K#-D$a?n;|dCi8oGG5BjJ zwG)~Psu=R>a(cNitO6P3Y41nFX`*9$tzrA(E3i^BO@!4LpXkkAIT1#Ws7;?D%Kl! z_xGK4!bb0Zn#*HhB$D?r*54c7Pw*XI+ucP7<|8YQzwEuqu5*<;FlM?yq4R5~4Dy`^fs2f(b!oMh!5TW>t~CMiuF?rZba+6}j226& za*O2(vDG#9@T8j%S?O1#yIiaueGHW+TE5~nD^_2H$U+5JND7m4zQ{(iLBiJl+_rq#M;_Sjn(*K$l3hp4c8vbgZ?#80x=PZi83O z^38ZQXPgd!5rzUP^lXy3D9N^wY~iXy`;_8j&R&eKuWHj%$Ur~AGU#%!O#8KSu*}hT za2LxFKmRDS?Dgo=a5OmmcFg}7e))Q>zT2}1niX@wglU0|K6vH=+Tz~if}z`7t<@ag z&}%N(yW3H-Bg|%^7*;Jza3|iRrQ4E;ZhyoZH&-nxi+A7RJso}-?1_8>ut6OovME4> z%J4%59kq2PK-?qo&Xr{jM7&@Bd+_mgI^H#Y&)A7!n+>Jlm7fbZM%2bq+$;Rfg=G#z zz@LvUumAjOO9W+|0QSYUs`W`;*r}|@U%hloC(|8MMWVYEIoGK8pVlaf#-T5gN3~Pl zM^D!C>IACqM9oKg$E#b;;8Pv(0xi7Pq;z1#rEM(ij~o@>VZ@`O;^&uHS?FxNx7+?K z2=jnZon8-+YF5{+@eYKdz;0xFBOrO{RmlrM8nTdf!4P;>S*@LS0w{(>ElVdv46Ya* zRkWH-anck;g+0CHuvEBPZ>hiXjZ7_gR(yzoXVpoJ4O5SjmuTa?wuV}_Nct=VmXy|p zh|3ub*HltkB~P?X7Wvp&>p^j@v>iKZ9&8F}OESuz*Td0OEq*u?qJG<{%_-0?gUQYC zW;z&s99*2dxcvC9!Kl}^YMamJ!(<_apC{AL<4{PlR{x-V&GLT9`#fRJ5VVy=F5uo_k1}F_n_wWIwUH~W~ zN=B58Bg)O?AA_>M^ryk}{+Hi=XQSqJGPt=bH@&9_)37i9HW^<1y~W~bC-dz(eaYf1 zul;@cK3et<7Cj$p)fx`UM(0DYk)OlLI+J+Q5WVzIj}`%}DNDmRWweB*Iu;nOC36@y zy*r3@g~GhK7|gDU!IXuwixiJt*UpxmZScX!j+57ANQnz7ki%$Lm?@aDI6TC|37>9R z8=EUEUzxldvYbPf_XTIlK;?r8PPWDg7CGj|@h^DyIpG z=6FnfP3*C-e+z)59L1{u$QE2AX8l%$!Ic$frNEX8wrryyMeiIssmdV5!CRf#snIOy zlAXij(36ee;!eF2xm96(!8xzA14Pa1h}eKcUT{%cr6C3bPErVUn5{QF5ar)f?$pm8 zu8047J4j$>I)y5(oI+)CehtQ>gy9EGuCC^7-=SaOjCx^g2R}6cJe~ zmhp=k6rd*S(Fb%`q_aCZoIXP>*z{Pcld&md(*a;prHgsl*5~_+lh^{ zk*uSK6pG01~%a*HN6tjqfBY3Z}u5$iJ1YF`G$ zFMqJ&;$P0oHnzvB+EM}X-BgR3Px84J2FqS2#?o6MD@9NoTYg{Dl z;Nj6n7M{E9jjf%vGjODg<2jYKT*iYGK_ux7)t5 z#Z_HiIvs<((-@hCdTLiti|VgUxo+mK)E95he`TFGp6-Do=__tj`HqIy71jGU7KC%Y z^!{t!ABP_%myZT)@Oeed{(9wokg@(+ThOs_+TD}Oda0e*II}$UL;bY5u2$H%+1aR1t)ooq zn8pn)9kYn2vuh;aav5OGh6xj@)0pC5Yo^it&{;s{^K$ZP<5N9U8bgnEHhvmjU0n~3 z_VIq548MN89=y0_`>9RlnglFyxi;3(vU*<(MuW@g=av5C;c*1`$(k0|!hW?4Ha`mUI-DCY@ zq3GXS55J~0Kx*2OihTv1*}Q=zI#nA?cfRtMj63|L|8hJT4JPkc)!w!|gh}6X za_^*i!*$od{mACdNpYyUdon*av4c_^Y~4k2!lV;@Ykdr;z}>4&nr_ylmWf z<6`v2xT3(ARpN@`15Rt8RJ*GtW_dybAnuc@t%90#0F!DpmGL6u#cX88i|YH|3|=&H z`F4XBDSlD3)^Xwlj|LoLM!m?#HyJN7UQ|YAyr{na&EQ4c7+wUcNFpj&ra=e2950q4 zP6&Y3I+yVx<3+}ctLC56{Y&XG&_a*gz;lA?gD_jy@bseM}iYU`c{ zTHW*Taya>z-40#HWVgfq&qj7T?3vr)2`lYQX|_Z=x7HAyMiGrw`?8^4RQj6_+XG6q zLg`k7*=Yp?+6UQ~qu^MBdn&f+yf&=d9WANB?2NLJ^*l)j4T?7KU`+~Rv_RJDAT48d z8nyTSryqY9eI87P)63DvWEj=H2JX!-)BbYOg}j04=2&_1LRx35+I(F7XYGnItX5;N zo&}(kCJN*Nn_etX2n8P`y@i4zJ0)HNrTP+e(lu=5DNhhF$OerC6m&o+zKbkSF>0gh`#W=G6G$08E%5X)vVfG) zV}MZsZQuzFe0ArtuGzY-X_O<|nyv}Di#Tmh*`)dzWo46j0srB<50l$b`O8gNo9El_ z_7rB;YWLA3_)GNsk~O+|zYm_O$M*XGiwXe)1r}8ed{?rQrWFQoq-7HhOf<7nFNe|} z;qfJs6LxvJEV5m-EHY54fKx8j)^XD{mulNPRnS@%xl%G_eAz_(N9}f%qxYjG8^pDP zyDX71vZJkuA(wm*{Iym=0In*h>&Ivi4IZt8?YD|gVssEEoEP~)DS=DQ?o@noPm4NZGZBgs!>FTF z7$#Zs7_sn+`N*b|!@H&ylIqdeJIjo1b;cG`Uhd4;GGjYp#`e$g-uJ-jOm-EO-UZcyedTYZiV zKPPSvRQxa)L0DmA2;7>;_0y3c3*6%cPN;aFV&;vqh1rPK5CzTB1UP^q zs$RMsA9SBCh`EPgHocw{Sic+ak|1Zc1Xe_m^Da~E~XMBRcE~B*&tSA zJj;bD3y0%=ZV~uYPr9tivMM`TRrbqd{KxQWaI|eEA*O8HZaOST<1FCUNZNpdBLB^T z6#59>0Sg8nYfZE^AWk{}CvAGYLmeL;o&COt*^7&(8o?rY>%`d?q*cB1Uhx71ZOMS< z1qw&mjEfn;{>8E8W?an=36+?eaph)Q|KHxV|2B~%;rn&-SNNwBtAn|IKlY@PLKYS{ z7T6@f9YR_;@jxtWPw)fC{`FVY6MJ4gldBr`pa?&`0qx~sk_-()L_adCi7 zX_=;+VTyU`JxUU30Yf4MlLmJQx3(A;*L9^yLNTtuis@;(v!fAMF}I8rGZf=m?!!Yd zu6)}DjklK4I9|QaWW+e56jZumC|-@Fab$!m#LO6CxwO=>+qy#_zu(rWAIR@-6xTL| zIBO}c15SnIOcFwrQA!iZai>W|QWIsh;>HMTJ*Bj&m_=|3z*|Y3^Hmhr7W@@j&JOu0 z;6=cTT+4tL`ToBdyjV+do$KR;*S{v2q<=31%e74k%(taXUd; z>)FWN!b)7+R5$6a+;#|+xWb;pZjDCRbJ(&yhkLBEH(!a1>k7TqvX!{FzBYxGR1ui2 zLJ;tIEt101haDkBL=067248~XoDw2fq#nK_F~rv=M0oT#8T4MC4EkSZB^{iKyq6ro zEKQD}*OTWL!kSy(JA^fFh8zLOn7K;Bu8zmh5d+7A@Nw_WS$DF#i8|WHSJ#f*ilcWU zB;!glEfgjyV)_l1b>MH;Fr8{mRQ^bC|3f`?G;E8 zA(&$sVqCryEn4qz*6iI-!9LFzOfuv69gVgM(7~1Zg&7+><3;mzdB!Tx1_pI}FVu0~ zGd@U0&^fcy?RAYCn2iWY*|MZEnpuvX3AHpbSRI`bN;xrUadhb`9I~Z=S%XgNVId6Y z6wqm_(J6Bk4nTiToUETU#EZc36lm3=UeZrVWf`I1a3TA(kAqAdlP)#~w8v~a^TT%qWmBJCoU zHfxJfCDWx0qln;)+Iwb;i2%646kCHScHEtO*(FQNiwpP25)+~FdN9Pc#t@sG_qx;g z&+~Bt?>Cw3pfy&}!}T+91zSwEj2>TQSrwNM>z5!4@UnOzBd%B+_I`#bcT)ScT5hO= zb8T?LN>XXSn4=m=>MU2nQbjeb4Y9=5Mp`PE=28&tOhHT{M=}~}EkjD~`?yxSP4OM; z+Y%7IfP{8pW~D2aW|3MzF{PZC(Fp5?8)cX>AZ+F2m|SZSK}%@}2%03Cfa0PU8|+$J zqd5U_4Pv%YF>9z1Pgsg4jA4~n!eRKd1&~1C!PHd=*pVi#nh?sno={$~o^owrcfR$# z!|wd1LwOe^M<4X!(?J)jjDCxM?o3=X(QF)l-1~cXc>b@>Y0`^34@Sv!Zddm@{oyd~ z^}&=LTBq{@m|jzG`fIDqgVPytYK)OWDryx*4l`vVup7Y!AVLx) zbx_}JTHFaMapl5HLW`=H5gA!6t<{Q9VT7#>F&D&ETQd_eXa(&+&M1{Mh}lNPtlmll z7ql{bRF^QuRVBnkzEGEb9Y$g(ju?!@duAm5+MRaMNc_0>U~(}$Nd}Kjx|6u~^F7J` z5-V4n;)+JZg0n6-g};&-P_L9lmMFz6U^9`1)-T~u8F4AC(#R8Mw8F}=l1GMW1InE% zON_aX$=DlNkTE)fsbn=w8KVTpWIV=f#8ttZ$C6Owu2)UuAx$CZxO<@EG7Xo^rqFjg ztHJonc?okm5ej-pXwDcqh~Ra&g=j*k!sQ52ODCdS>xF-UUdUCgcQ}K2gV_kH2+hiC z5}aMGNl2nKm7o`w5gIPj*3y8Y)xzbn1ev)bGPC-oI&(2n3!ot`4U`pzoz*5lftg%g z2)UwlFqLn8@pF~%4gixC!aF!$*=d`rJi>^z5(^ilvV@Cj5`LD*;844wI#+pGQer99 zE;!_h)O z!7L9#-eMJcGlUOl5=nlUVx<(!Az>YJB~zGvrOHV<{%>eAnSQES5zN}&4#kf2F7<}Q z?#i2o#O^gEcBf2| z7aF&b_~PT<+etjWInXcel-7$$qY+q(K~)f~h0!9AdZoI#GP<;p0E3b-t0*UgY23CF zCQ?=t3qLX z`VwYl^#vMKUR`+fqHVpxt1OSq&B;2PR9gL2& zm0-~c!ITkF>f5?Q-j7{;eM_ai@rmEi?(Wb0gJe94$A_S4XEh%|{qO9JQ_sI}In`#- zvKpoCPUR!jvui!k>h5H1#QLJCH)_4r3$78?HUv18&7D;MNApOzydF)E+)}s{crGHz zxKTLM2?r=@f{O)Y2_uB!l+m1{f>{-sQytJEphZ9n-%q!N7V5UpLY7I>3U}}t7sBFR z48i;>B8U|ME&^NxxbXgTTX3Oo3@)gMl%ZUp!9X;Twh1m80Y?}Gc8BD_7YT3?;KKXU zvbzqB=pMvJ2&tvkl)E&4?b*boIwC|H%_WyqDux%9e|6h?4rcj12Tu@c)+}r|gbjx~ zHyUBXVaqlg?y=6kDaBSa?>t%|(+QPYI}Jfz-oUGC);7F_z$>J4B7zdDBsIu^_^@LM zkBm@88BQq}%G%X0vmhRP?d8jAtBO_QH5@*5Kab*~3sDM{XcXrB*+&hh>BA1GKl{jc z+9Or%p@N>qLGpc-3qdBhJH!YTD@?546(b{SqTn-JTgrdVi7&!dO zMjT<48q9Nac>#n)Ai{>gY@=3k>PUB0q$pem#lr zn?JOVE8!ZwQp=m_O>0h@T)&D2bi0oX`_^eo>rxLKJAel`o1MHf(`9A6o zyVLB|{om*=J?)QAli&NJU;C5n)i3vPOty`aU3(OtyJEG;sB}4rNa^~4d(9`K&;3#L zU3$s1@-ny&x6A{k-;?uxtLw9#bp6*KeW{b^>IAs-A(yy*0up_67Uvw#j1?fgB!~)e;*G%Jb#w#i$5O!@%-q?+5XwY-s`vY!`aERcY}R6fAB;e zAF~%k3G`qd!_?s&m&3mlMefz&u+BCj+677xP1}sTz|bD#W>k& zROeyb?Y$liF0P)jhspPXIoY7G;nJh??kO~0TaerC`jDzmzTX#~> zwP}kXSe))P&`e4#l>Jkzpxy#TTw~OC-O*{hWZSo+{?}QYxg8B(|8^2T8BP1+rfz@d zx0QJ1X7Br>Uh=&j`wnK~G4zD%$xM`Lj7B{D0CM#2y}yMc9N`E@ aIKmN*aD*cq;RwgwIsOknlI%19NCN<^a|Q_j literal 0 HcmV?d00001 diff --git a/doc/diagrams/jobboard.graffle.tgz b/doc/diagrams/jobboard.graffle.tgz new file mode 100644 index 0000000000000000000000000000000000000000..0fbe33a59d11b7852c76042a5bb3b32af1336e55 GIT binary patch literal 27254 zcmag_Q*n;m0?E4FQQ)UnM@I%dcAijx%^`+2|rANzD4RgF5l z#yxRY%_571hnT%`)`Wn*=(Pjv%eL|O_IZ89^n8HoJS(|4ew_3K%yf&9AL+sWtzv^h zK!=AhFW32cB9dssvbdO-P}5@p=$KPE+A*1n`X$Uwk)Z}KHIUH;bCdeb1O`NsK|oL) z4h4!waikUAItP)8LYl#-JwsB4aM%!Yyh4g0@;}$3?fS91zcp|CtQBke-U>ANT-QV<_rt4fmgWZcyMrDe3azuO4BYm<6p8rbZ^d?9^i179DarM5Rn^bCCDsJd=B zlW}0QpC@-dJJ-J5CV%O;J@2w*Tv1;Gsv{%bYtbPk-QH*K59D-?C2NGlD99iXSsOY8 zK!v9ju|E_q+JGYka;|=TFFdRKrxgc!4qRx?T`eFK9Sy*}|N6m#XaR0PqmRJj)JXp) zEbtBD^J612P;CbIHo)=u)Nj&2R8a4`e8g=x`n0L?)I-4l63PGKsyguvDp?TNNImu! zxQjkG*|oM```mjuS=u}CW^F(%Tz&nk+v_@=Ks=i}=>#2kE@A?Aq#Tr$Z%=@v1I3y|T?K(M!?>100zH5@=cuXgXt(9<$< z(>YUwa3y+-5r(Xo(aUB_ZmBExJQa`}_FHJSaQ<*6%{h~)!c${^h8Pc?oDq6D?Df!A z@K*&ZvOXo#Q{E+8-cF1vq6Q^Ogunb`twumS9K21f`br+5E6utPzC%gDNmLDbRI|Zy{ zP<>X6E)F&ZOhKS2v#xX+L!#t(3{LG}2?`pWp4%uLD1zO> za+1*y9p(c>)cP=veH=*G1-|%dOB`3mEVoHUZN74ynos{>YQ1!Ly!~_-Q&eoo0~r5k zCMfc5D8Dc!#iEd5I%PFtKCPm`T_)wS5G#&UO(}>@Yp#84SK9t~tWO%1kbS0L|FE<} zvVVNLvW#?4CTl7%=-w3YuDfYwG2=L<7+$~ZFE9)ciM)T%*?0pqMYiTw=W z0sb_&{j#K}dw$4(VMZC^o&il_-E?Kl4xW_nhd>C`D(Zp#fdY8p8N8F^3pLoJ^PBk4 zLF$;r_(N;*VoRJ6b7Guua&I%aGfzU!X-UbE(xYN(+!&eSyuN!{$;nYzGmfD;Q{r0# z8Jy_stj?6M5{Ds74n|?Qzx5~<#Q0@lyRiv>z84kITW^|0A#@VRywFgIiqc^ZMwr1k zV;8lAK^VQ+ahP%`8buqV;zeRe=?b)-;L;iv2Ms&IP~hqiewTjyojNj*h-fsBVfdjVNrW^Cr4fVC=K@Cn-8dH?rZ&N5hG-LIG9-xm9iK7ZKu0ko zcTV@O-+VR(FRL;>KlERGIzHpUNbKn2AKwtr5o7ecgPuJjE*O@OJCY1Xnax5DH&b+4 zztO*py5@R#3wzDxO=KxVLj%bxh(*isVWr~!I^3K7_Fug`k)uO#$6!+qmO$`k_M?QV zj0!dSh8>0X5IWkzi zwP;_G`sJ1RR^aGeNSJNnAK2M(McmHIGOkmbrXoD9j=85KdRtMqk%<>)AZb%@%w1|d z{v!nT3t#^4k7Bjg{HNS0c`NtlgbZ-{Yv#kw+0_tHuf zRkl=7^{ZbmSIT_e>R7kGeox>1xH_(-)L*HNu1u>*lT@M!AV;gs%i{|$#Pz0-=h8VxG-_U>l zkfSf%WJc}q&_c&-??FIUg!kp{OoWn??V0npGXb%^{-o+1f;Vk67{$!!pd1cm(B5zJ zf|aXBm&R4W(EM99`I=HPI`0s_F37F}1lnTWQbSm`2aP~bVphXqXDagIBnyVZhuvhU z5iCVB4dd~DC0T?U1`;4C!%DC%|1})M=HEf=Y2s>R|4#k0xc~A3LiHOIyEaqVFD=)3 z!an?^x%;qIG#c$#D+%K=ZnG7d&{1hp2%{B{IYrGq2P_6phw)lgVIfOrO`vp8zy3Z( zImXNE(-2#w0KL*qt*kl4P;G}2XEH?~K9q5q;I|c|A`6Zs-5y&Q?#+Lt_Nf#qr28 z1DJhnS4o)UbDFDv^DN?}h`l+Fx*a6&P>@b5-(-uWBe#n8o&#rCNsgOY5lCv_Lh*XW;Yp6|% zqz$@PV3t;ZEyb((YtJm1OI5}xe@}@BlbF7OF~YBlx`U&stJ$pPg3RTMzlDg%(QT#D z1s413Ku#fpy|av7ERj}N|5tJd)5*}y0goVrUR(aQNgkHdS1=~?s2SfSGKq4vT7bg_ z-sCOEbpXX&O!*S&i&s-ROP#ctkTT^4I^fg@7>6jdFEIXf7dmeH4?eKi4{v&!L3y-Q2c7M4Drp)=#caja?D{CZf zfj!#A$o~Ck17+k_=w0mV)n^Hj0PHv-GW>HEj@|>+pPc94=y9U+E?;((5c5qJ(vlT% z=lqaiR##g`Y#VbvdIAIR-6*rbz8#E*meA34-lT-h^n^rc0F3LDbGspeq&{YMk8hUU z9xo?xo$-O_4p!I290GjbbeHLA%Wxe>XjDTV(ZkapysYR*DaLu8&AO!@QaN$DG>EKu zjOwqyNYV}$M=7UON>HF-q#I2m>9KceWD&8_VCK9V=zdEWJG$X!*whnAoBuK)4>WR5 za^f9oTyH-7Raqwb)^TLSh)*0Eu*#aDyxvl+E^%lHjez4S9CDL&aa~KfqE#QIBmD<4 zYN_en_8;wFJx@ZAZq%x$scGD^r@`t%3eJc$b2$kEn?s9A?ER4SrS3Hy3umBy_AJPD zB!q!qwf>1*Mrf0YTGZtJIWE1DglA`S-H~Oe?@ro||4lXnCXiS$ z;&+r*s*PRo3MkoaIlku}PlcSm9rdvL_Olj|$sRQRE)^pm=c$^WQDe zgN0ou_%eLPl+iE}yXiF4-zR+k&0<3JBw>jm8(9~Tt> z;Dh$1_xMnQMxf-EyqO~LDgKy)j*SIWIeeBVVhJJ}R|Nu5lv)UJT!9?WyAfKVHraRu!Y}(jO-rqw>EzNDc!p8Ua7AXGxCh z&R}yZo%EU|fWa0Lj!%Dyq>)7RxfqyeQB=srIZp3qb-!!%Hm!S&r~Q4^;)>O<*slyF z5SZjPo}!vSidC#AP=DiA$wt5t&iUMd99Ug&2-FOOM5{PlN`X7R@Q z*+q@s-FhWVYs>iqTad^$T|(l|`++Mx-Vb#zNZEqxnp@L%sr@g6kL@q%kH-{-b2zln zmPsY>1lCv)me=@w#KCUp)@!V2)Jss5PdWTrv+J9kk7c6^K+8Ms^dqOCWPc?`Z?12dJ%u_)P~}$NkM?B)vz-h`m|gC*jNwm}K(I`BWBW2rrmP1f=Y_ zACYj|qV}U=E8D~&n!SrQnL1e*9-`=e$||XC&GQBnv@{F*%xom`dkKScHXJU(caEUW zUA9JE{U={hW{iWVgaSBQFU}cW%2)dCcY)^)J3}`cRGBxMn2+R3;kjAX?ZMNOA=FR#L_+6Ak2sNRg>Q-;qaIIfV9&t%99Z z1)=`@n}S0(5;GVa*x%42zCQ#m;ds7&bAkcxsKqd3_A3mPC82%wLCaQhv|o+giZ~-W zg%{B+EjSGHrv2hIINteJ71q041=F`&rARt3(Y{dil#W6HNK=~5Va`9NeLN?9RLS0- zOV~nwTWU;nCPnUzZBGdB54a8={_yHz{|LDLgy!hs@5r6TiHS{U2S`MJojwQ1Fh`*q zJVx&2AV0qChyok(YeBxb*E+L>$N_<$ERAolQHK*lG?~G+9RaA?gN`f=)D+)_Rv7=7 z{B7=xb%xzZ(A-!2JCPk#>4$EzbUV-e93bqw%uPMdtrXatK6H!34|7kk`Imq`h@8+D zQvJj9etzPlGOORo~p61}Ps-4Ez(0jBXb&5TkdrKYRPF8I??uZ_@{xDeXH2{BiWQ z!oBgYCt&&Z!G5r-_|VoE#xJOa9T9Y8GWt?OAkLK|5PLK#r;R=j!uDZ$^A3At>Qj z-||a)P+v9cm#o>gPWBV&XZe|%`vlR%rQ?Iw-EZgLM_~6%1DcO`@U1VZ$vBv)z-h7% zOn=pF|Cu=358BhQeQwCVStg92*_M5Z{1OU$hzuaSZ&Nuis{Po8?<@G1+4aUL941<3 z&Ea2b4GC=4IjCN`$o2iUy~J7H#bcb4R{f)Mq4VbUq~ou@^YtVM6v^O$|iK3TUYevitn%I`rbTafZy0lpoN=%9{Jqk zqFj~mWHiS+DCa?!_cNk)I&iTp9Gikl;-$Z#;Fqb`V39%aUIy`ApOb3ve?20_hvKwc z2ZRQKg*##0uYInXf4gv)S9S6nN0|WS54HYfGO55o{Nixve1woU)>7JD^5fHMhjcYh z+U1k`5Er70JUjuaJW#Ig^TO(mPMeZLw`T~*2+<=`BJ=tA4(-Nxih=F9VmNUdMZlT( zvc2GgbG-(Loq+u;&DZ5f^z2V;rbq^XkwqSoyqL@UH+h%sm1q1irb%TH(-jd*6#^0g z-lXPw1Mb^(=pYR%0zB_Q=8Ce*{p2qE4lYk0KH-k2Xb`MFdYb!RhJECihU#RsTXOOv zcn)~k4%O2pmUi!fFgd%72uP$5zOT!n1whDdLwE$^j!^z}Mj=w#^YB7u)+z0`09S2~ z^=lBrEOQ8)+c~bVy5e?GkaHfH|KkhW!6h4fSz(Gu0RZXhY2Z~Z!Er^;iSyszo}yk$ zJ?v}BEjP(WJOXfz!$u;B&KzOM6GavJ}#UZ~;$G02mA7~J4pS6Ezlli={I+mCUu)xhT} z#fu!`tx}GOnq(EQ!6T&Yp4JPn7V*^4Lqu;xu>Ur6Cm{8@F-d-U+9PN-zq5^ywvFWf znN;aqbQM_R=1ryi{u5Nx5PAK9l}3Tv$DBFAXC$+Vb}LG0@iPpeY@ftCP@OdhinUa) z3o&ER$eDca$v7Z;SMU!Y1gzrWy#EH&K91abt9vQGheR5Hu@Qrp(lhVJ@)sqQz5bxP ziSLK*!|>kSGgb*hl!3vj_bPl<{2)KzU}bsE7uLn{rn~-R8hrccf(T5)Xo)j`9F!=I zl9pzu3vvOwhI1nP`W>Gn3UP|;uE-I-m#O~EssB|?D|lmYhi<&r1V?qtG%XD#B}Hs= zM34dDeK%zwFwW<_jR(X@R2vkfSCa1C>;u{w?Pzy8ZYGk-k|;YE$4DcKX(@-f%Z=~d zK0%`h6Ww*`clB7~NDIrIR~gnzjc)cjz#q;FSJEsTs~ zbIPZj*YR%!X{J1S6FronWN0I7L^-JxXjj0z;;#ofM75H>VE=Z#eJWUhBZL} z`yCk!=5k06T(d0QN>a!FK@Ngj5`N2vh$rXNik;Tm}qQSd_ zbPR%f;ko}OwSLOJ9TpVw+kv4E_CZE$xV%WgH|T4Kju3QIlUc-r6&-Jb*Ei}8ITJH= z>}SvaFGlG7upnY6d#kCet*9mO;I2=&9ue^)>HyL!Gc66 zqI>C6x<%Xk7BlrJlgCGnM5r?Ki_!CXW`dwIk~zWO+WSQ)t%#AWT4t+gn*k3^nNSj&a|l69*3VL>!8p0?<^KkZB~G9IDZUjcAZGD-RIHht6JrG&KC-&Cd`; zM+?>=0{=avfxQ8n(WHhvv+1LMX`-^d4ebP;Y;B$N{nsshBv$ss)+{>bI=m4#3RnQp%2%=Ahbq~v1ITKxq{g;iE(;#{}(Xpjr9rPqbmcy z!<%0trLvnp@D{rl5C}b9AHti~!~nA!h5)r=67^Z&wR=mJRwJo0_fZ&n^~8j+=b)Di zLAhxj&3He*Cqc6a0R_Q`J?c!-h5q!^s4#IQhq9kZ2rm7WoVN8PZw0`gj)gn%>y-y}neo6Y~SDaxUsb zB?qAC5dQ{IF|PXQxlkIB)j-fNyPL@z2JRFJjW>O$yGo^4z1i`5e5SKGkYfvzkMWb| zl{C=f)(NNXKvsH@A6=(N_fwhFry-?_!Z2pGI;i}?q}&t4vt&>EKLPyzl%mu3bFca) zVAg*U470CZS{CK*6RvZQ(1+z8D8TmTNl?0~-hX4z$p64am_Hnm779~hXZXqQHCUYd zl27)30B;6NNVdVtu<$^5N7@5ygqWK$66ATfF#ZS$-#%n{UOq3a>o>m@8m;S6>T88~ zfI6mZ)ox%hBVPqaeTW7r~$n~aif_~Gy2 zhy6i*R!?cir*Ms>x}i$Uxt!|<)>dxaUmfd%t1eXt|3|@pWn!XY5g4UzPV5>mUl{jt zJmAb2-6hBhrS_j3ZRPaS^4_o;rVl2hj;T2uhyPv=TCj!hd*utTh@tFsn_c;J z&1Z0E-&99;cqu2w>nE|@90%4p0TYaDMD}vT)C(o(WE?h6paS;3Sn30iI-K1uH++2b zth-4%%ROsJ3Xd(et5)UIdG7PU?j8`ZDzx?||IuC66D;4Jpk+NW53N_kDH4pKK=tVaBTpz2c zzmN{IAwI=|1`J2i&|NUE7@{nNjLzFkEeMDhSp$lN>;=*MUmLK=xgK!M0s42+u;Yq4P z7m$$BvKL~?&lbqz!=d-B=vqmXwVB|75aODH-_UxYtpaMLt*+{O(e*@z`6;tz?Y^vn1 z2|q$MG=s+KBc-1aK%KXc2q7!Y>e7*x3NiTIB+K}-hkCAXVhUKOGa#Qj0J@Ve31e5c zOoN((hSxb(8#B$y?1W))yC8)~H=#!q!$yJ%e1m}W-^~+$8OQ!5HVDKozv?Ha`m=a! z&bU}s$o>%5AR1XIH0vNcjfp__M^wJyI#HtJo(fOz6s{O~{`MKR{QLhsm9$4+_G6bP z{Spcj{GlMmPFV~&G%OqzF73o0=q6DNh40^{L2PLJhdil8vlu)^T~0oPu2E%5oj*ht zQTgWglQ!$sP%&U|fwSwAxWG2So;-CjAniu{h-&;WJu#aYf-D^IZp}~|i}=uZSC0G_ zJC4o2`tXWbWAB&yHruW-y~`OiM`)6g<7@T`cLbS4 zESsezQh^BuS=VYi6K;Pw^MRA|;Mmik`*e35vwu~{PuA8&^P^`KNW5K#L{CvP0Bt<+-wODm9}77?rVm9vHM1uJ}6s+v}x`(M?~LX~K3dF1s9 z{pH(cV8T97!Lz@%2V4aP;5_FtzB>cdU3u-_IdNVBw17IM(HJRV}3VkJ=q>s6hWumCc}|Sf$|0xb|gLM(~en^v}EZ zB{F*#-|9a*OMBq>-E)EuTEXrfg9n_8^!Z^#|5}eV&;S~v7{fJCt=(EFW5$Mst+!v~ z;Ot}+fI74z64agu>;XJ)56(OPru@&PABkeO-PD!~m5>O5u{{|YK0cok$$r{6bZ*nW zFWu2e?i2>1ZGN4uXF-`PkMxy_eiR(1r+M`ScEqK)@FQgQH#fysB;GfdYy#XO4;_Sl z)vHE0pTPV7_|bD4Y7Daleij4!ih*B5!9Jq(7mMv`W`$capU0*-Tq*sR$8)o{?tO^e zCi?{qD^b*k#}p~FQHtmtUfv4Yh0v!OX0BV>12tR|BItP-kmfBjJkDTeB zvtErC*~0L7z-6Me7+^$8Q>{9k=yV2=Q4yq%zZENr3kClDy6^L_3&(__LT$>`QtpvC zD7zy)8T2!9x@SHC!+jXM2NP>8so=(A)8YIZl-rf^RK*w!m<8lzz38^g;? z`p{B0=%>%L{Cv3esRAIak6*0OH5Wc~cPK@M8&mWQgq!BHg0#q6ka7f>@mZQp3Q%2q z1Q-=*h_76Pz&OVCXG{!L2D^t}sNqA7#pTi|jEjKVPjILm~UNCS7d^>-_d_2o2ag zj2WkSd(3L91M~)xF%k^f;t}5d zt#GFdQK7RQ!m9pLL?c>r{WJ(-wSGPat%mQqJ^eHS7N2!W9&Hucgv@@(!DEJmkyFV2 ze=H{9ML#X?9Px;&oAt|t=7k&2;i^3>pzM9c^IK`Txw{inK9Kt7r$}hET<#-~h;;40 zWw>R_txR8RAURv$#c9ZCAr6H+>Xx0h31czbiPNRpD{CjxkgB?VhxIew;zDg!tQBt4 z80CqRr!2EK**Vge3{)#>Rw-1RvEdLdliB~t&qx?25UE$E%Mfg)33rBrly7Y3Knh2ot?$>oUf$r?mHp|` zeDida9ewIGa%4X|-TDI$YKe6H40i^X(&G4`NO!&Rx2hoQr|-I8yH2MOE^COEs4$}w z+CNFbj7DWM!5#U_30RrqW6d^Qy(7O?--Vj?rbv(G!A=d!g|jP(hHcG-T$1~dw4u+A z1}=k`4FZCfE@w-}f&%7H{YD$&ChMJ5WB7>LbmFhaI~P=64-qL)tH8N2EmK~_Iy2g0 zIL4Nls;Z&Ge&OJFjIt0szuH#ABCV)_^=yL3TcUL4p1SfIgFC%lnH)|=FNXTAt=f}J z1+s6>4X(PE+enGv>hOALod_nx2@2wNORK7^+y=2kNRCJ{U^WyusbU0Q$`>~4ve$Yg zKNuzbPm}Y_sj-LGY^!a7SM`hkh$+lYCNkE9Oa}Y|Pf6nY9EcBUg--UM?VBKDR zTnKLH4~@2%4j_d8UKw8Vn2_jwk_oWxCrR)G&XLt9T4Mh|ijQE{zbzFPr6&A2=>98` z6~CDx1?HB2(TXGvcSLhWWIQY4A6Q97#sG1xbA)7J(g^-mShsfflY$TDs9bF=qXJh7 zqLF04x?z3k+}If<4HU#?FnL0tjzWZD0CIJxMJ?VDuy5m;$!K#D?iS(ka3pK#KA-P7 zFZli$Zd$F>{8BEemg!*!M`K(xy#9N)l<)wfBCXIh(Bc*H0o86a0sfY@={`5Jv) z*nT2PS$xloZxJzK@17<3qp_9cB|Y-r@E|yEJp6chEwYkiPcJW$w7Tgo@!E2|NLMf6 zf4#qS>eG59)cVnF>1yLOx`*OYUcI?CZDvlji%^s!o5pHs*-~n#|Jk_IbJCA*?MP(gvq5lB@vw3ysSC zmgeNd>e9AZ(|tC8gMly`l)&_DG?3lL ziI`$0%icSs3UF3Etm(1Ry8q|=iSFBjLvxuYagqh6yGYvO0|m1Q{7me-I<`V&c~BIq zhMlH+yj<&?!O=<0?~u*w-BR|tf|(b!(y=_XavuTGS$$XS#%>3-ty+{R!Lu%FYdMj8 z2e_>a)eFP+l|)_p5<}nkv5tAh`jx9{;D;S6?dR1plRC;RlvRYsk-{tyxWnFe?qCJA!{zqGJeXj7F zBFN&LIOb-$vtww3agW{Mhjt_<2sKvgGCktc1d;}TDjLO(TW$2WE6_?b?CRY zA)m-H5^XKz)3P9IvK;89Qz=~A>J7%77!P^2Z|M|VUIhoJH6iNj&7b&F^RIfJdip} zuQJtkR(V(R(Yo^atx(KbSzb=pBX*vPK8-p%>du8k&3C?hCz_A8ZBrB7ij;aeI=*Wp zTlY9T%|;8<=QmF3&V6FsG3Bnjh-qa z7@yDfb{__i%>R5z6|>W~$|FjRS6R3_GaGSiUDxsQ)4t=8IZO_AEW?~SGEX<+@;5z6 zO=#t}thTWTp3>woBAVSe{qQKiH{~Q;4tm|l$(PugL7d4u<-@6oo!RRX#OM@leox;> z66JDBWeT*ozi_@+&Wl}e^V*H>9BFOkpzT30|@ z=9P)pp!r^=ud#qhPGJ`%{PH;ylgHgSJBl+Mebk+z;`3RQ>fU?pAJ%P7ofQkF>hCd) z-w3%s7A{RT2^K57#5bfS@Z&`qQ>v8;45R0YeecW07K(%t=8+;9Z~vQGY4@|bel`NM za0!*12n$3P8=xns0~*lsS?nI-xBb%>R5s>|08*iS{@!Uj@??zWc0~~)ZTB6wj;HZU zD%zq*5gpWp6*!F*jO#8G=~jRI?q3eWPJ|tojpz-Ue-xrc>?qiLa1CDOY0VuTF8`V# z_6g5fs}IG^QT|bPZ+@6wJ!gy(;~40XQlMKGMnv$nk1O0InmyM37B_1HckUit04|yl z3D8^VKB)$y8y(7J2qc?I7iLcc{5pBA5X_^~jJYr6r_wRchikv`ERCy|lIU``!OGMr zG4~U7E2QPRa6U7pCRa~)juQXR;h#b%1PhrE4}X+2Z);1M7(DQ7F^^IlWAs&SnRb|e!&h^Wz z7i+LR&or>+2y*sUk;)D^I{3Pqsu@W(6P*_XXI= z`VHPUZb!TLan@X0>r!bksOYYTNBi^L$$dH`?u^6n+BxzdfSU+$$&5~IdlE4ZWq@-? zjdQ)5a{IenCw2y&{NtqYK3mH<$v6)Y#Nb-BSL&L3iUxbTR+fVnC_>yB^N=#Z1eZ=# z0UI?R{Vwz=?YYgF_^$4875BbOCeQV3#rE<{i_0H@d&M721yV+v4$)))cH zn0#u0=t4;R9%3y0dTrXJ#gHL=M`GT+yeFzsZ~vXlv~=GeMh&`9AK21agEGs3G5+!6 zB`!g5_1eNJW1c?2z8IZp38;t*b>fA^0Kn*e_dDH>o`gRdu9e5}@wRfN!{_#A(MBrm zl$ny=%nzid%1k(8--(fUi7ptzfR>8-z{#x zW)UBtRUhY5t2JOmUMR9q+2X^~8&TRVyz(xq_3g4H@7P_To|U+fPiH-J#s`KOf=7^2 z38{MKSaI^+z)4=JPEx-yRC-$^HU)4yqrdQgW*>NgEf73{W;vF%Wi_PUmU?Qh=4`8% z?XkwktNQL}zb1BU4B($%Pgg5yxeM={?5ajNbq=TpmzVU(BiX0F^BHL#OJ<)<%0x+eJP+l<4W&*0kmFYKM3n zujcC*`BSLI3?H0zaN;sq8Ox=&40C7^3obPm2zzAIWH(_rg4}+ogUL2e2y#T~p7ouN z>NZP22^iY)J^D|ED1bYA6>(TcU6hES1(_9Am=qyt**y~2_L&u?k(0QLIdLY(~lc=o|-7~ zxl#mTQmx`b+5^ z?wu1#i4XQH+ydOa@3gu;>&9z#&9cu$?pO5Tm0Bwf4IJ>=vE!B7q+H7+f}-t&D5}jL zI5{fq8qrM=A7k<5)ie zsahn`E6~(~JK%5oxw~Ww`tnLIR?%?t_Y-J?*&l-yUN@%~xx$Hsxh+4n{Ya+C9Mk1; zVy}$B0Dwb$LX{(9PUKj(3Bv(*ecLK**obCsrG5`2NMIrX4n;ZtV<-zBbLGYMt4uqA8fi2Gxs z-P$;?kQGHSKoFm6y0@{)%X^q-$(9s%ajT_PlC7B=dP(1NDx5=Q>4)li$wSw&zQxW_ zdb*=E8A**~`I3C5TLl9jwlA>K`996CI}0OzUG@0WREwoTePR3-ot^tEUBso;tRD5d z`QDaXx5=udd!>+=^^BTZHt{5{Twm`NcG-epg04MsqP^q*h;m0MeKgx&zZQYTlQH4H z3jmwnW*JS;#XfKWH8N;QXO(ixWXg!JU#fWrKclU4Vi$UVd`*P4by_a~&5k&({*n z>nX}+aawu2V#rC0H#!G32lj=ztg3D;7IBdHIEj~RIxsfP0DE-32iOQ*(WHoDv%O(G!oKPd9%gmfp zZRm;wZXstF*1yAMrE#1L`;XxQGqWD8eV44JHn}YE6@tk-)su?sCr7Q;6D(x3p{~^1 zM$ReI2##&zM|-r*Ea=nYOwBVVe69|WpJwi3EV9MPGq;)yx7Il{W773Zn_j-!$|~Rb zI!)RfdL-WBi_Eb|9EOa_ISR&YU2zGpwIm{VS#0KWd}wfynZ&)S&)Gw3 z+A3K%9q0(PttxnAS#?DJc$M|#$clTMXBXL8CvIr*<9mMd7Oa$Co0k&#e(`4XOC%Zm~%nh2r?)0OzY;x|<*K%}WE&>=zUdl7YaS)oLx{2er$VWa6&4e>rVH2>! zrN`U7z|KF^0R7NR92_ny~JE&F2Z-cgcpRb z-g6=bHh%KTuO0KkIJwE8WoWj?rAv&cRf}Gi^Eqh3cB3RU^iiY{?(5mH}-F2TWuW}c?xzH|BZ@=#3d0wvoLeN+k|Fn)=?P1;+J?EJ9 z%(3Tedu=l>(fvwzU5T^s&a%z6*p~Fokeww9kxfxt8~2I^p(7_fx=3E(t{eMi@eHJs z^`%U=9{BW)zCO8@H{zpB#PHjOzE7ZlX5>pt6|NfRs|t{bm(*q za)4h;_0+ePb7<{3W{2bb+SHb)e~NRW|T zc4aFkj#ciB!MUbhr7MtBo2q2hfUA9YPe53-d-&FVv&IsSC$HH&5!v#j3R$3F%~QRm zm(i;va(T1rSi5m)XW*s9qULqW&i>ebi_5i?w6QJzzKE}Ix67`9eilHCN(2h_TSxJ- z7dbz?Y#Yws=G$;g;OS%Y@+>>_8x2`7V4Td zVto%-Gd_qAacbE}t}j+OFS}PfN1!=>0m0_^j9u^&hPGz*7_mfAaZCd^}} zw>IrcKqhQNTFoBVZmtnJOg15PCy-mi{uBm6QE$;N|FTr>NSek3Gk*dV1 zca>n$4GXe0{~LkQ-LGN|fIx;{A5on{tUkafrzXPBXDW zCwX^S((P!&^PowRW+a4=V9n&1KmVYn!FlNpI5=mS-jbR>W3Uu3io4t_zj6$Bby;uL z>ZJZiGmp>l_mcQ67xL2Mbwus9j5e3Guh^}DxE|4EN63?{gtu|&vC^ViYzsOzI{WoL z$eeThd#cl*G)KzhBpfl7d2tp_gkd4?UbIMfi~IIJAF8~uaOzb5OIFLibYU6KO8v}k zhC*;l?jPq{vvM|HsF@PKM}j;1zXTo-7uZbkY4i#HG`9x9 z;_)TK+3PvgANo7T$5uJ^bU?8!xrx!%^>K(2kax)9nDb?|v|0mPt*H?y$;meF%<*!m z)h&j@p}p$#>4SRhdflIyJ|dh1FI@#9v8Wr!89R;Vk4^P$D@eQ;Rx(bk_fy&G}bkSIMrGTzYxxVw_owjv*E+$A~Ze^y`X}0uXIYFxgF|t+7i09JEfjUs+HhY!lCdmnK zis%Md6DhI$2Q4y&b=U%CjP|g9o0mi8cJH#Ey1(-6xPH$c=G3t2U==zts4g{HFMsC5frJwFN1lXRlIpwS4-cTsBrb*M*f3|5F^jRE35P!4y zQt{;9$1CDqk9D`VctI#=18Y~RNOlVa$4FW?CXIVxSWa<(^_m@->gSL!Ey(JR4yEuC3luD|q#Y^M>Qk4L=>Qa%D&|&X zDio#Rs-V5Z-P>DiGkdw-YnD?9*(hRsaIYvvamJG(?)55`7t zAXm%h!#P_sEZ~R((#gz9@&&!20Ubv5M4W6^EM;Weq#)8})Yd%uvluY8(Ka>7#Z`nC zoQh4&I_kT4u1-bzZIRJAnHW_}aFvk*PcG9BdmzR6js6)g8Eb1rP?%UGw0OMh_*7Ffe={{%g$@g*&j&t8pt zrTrQG_)Tz89Zw0&yXnZB=EoE>3%uy%7F!sc)-m=XW`#=3 z#k-~F{%hLIkg37H?|$Vt;8Qp&_+3X}Q(r`B3NuVzB3D~9qmWEDS50E@k<&MMbw9$F zf0SVkc5yk&N8L!)7t+?PZee%FabBPLfyJu`7p3wC0_ExBveTV@P{~)NX+uslUEnZN zte!V7^_xeFffVrEjekNAH1xv78*|Pu)k>%( zP^oy?;plmeGxuY`zb+0>4Y}oqIBVHMIoC;9i0_=fwu6v?Nsa1ETPLHyB)GK5g7Et_sZnE{~qmKXeTEYJlMaj7)TX^mb_uo+h}mG9E|*HL_wK z%YroHUip6c*cxS)PpMoA9wSs))g@Y8-RB9d)RD%@HlR*}G-zr`(mgKNY;_o2N@9W-9U>_n*-a z%1R6$Lj`T!BKYMyKPHbuFn^1t!#{+0wcUB{q_w9B2>c8@HR$DOh-sArr-`>#OH3U| z20qCzSeTvXr%JZbKCD#>W<~nZ83mHjxLX(2{hnckW1|b~l3fN?amesD{hsr#DKz%_ z5zhU$!Qa~mB`(ff6Vsouj%$h(*{1hjw!`HNF-)xjR_1YddeT_Yreg=;;L_++ zpM^UX$g7>|IrCC!9p$IqV~vP3a$%2I{b^qZ*t?XZ;IuU8c}uIumC?LhUX}-SO6P@6CnE#T$RnGeQb|3yW+~gOmP9L`?>m)(VYe%V zQb9VSHA6iZ6B%(UrKr+$=-T=9u`G;=%i7wuhiikaQFj}?-VXLMCp~SGb1StwEz>M| zhBR)cc8=pfjBF=GsFEI67oHJlFz@1SqqEwe!JbtHsj?jzlvyeWrlb8LAhXt~U2of` z<7j&__pObc?l4{16_*-WblZ)c7x{cpMb#k`<6$p#%&UE2Zx(K<3)+nqa*IJeKg&$E z6;I{f=@{K%1f9ITq4|D4;KDcB&bZq5OQY=qD}B3oVA*r9pY4}bZQhSWW2B%;KhT6S z@Jq{n&9O(QVwgKOwXS3<1(H_#8U2`Nl~p?xB?epV)nHw=HtojnJkrxSXVS0w{n}I^ z`$LYFgZ;GFD>V4-kb_YSlrl=w;W;%NIlSYJ3Vpq4nG{IT(nJ~5%x<1or}ensal5++ zK#SBd?RDH$%I$Y^y(M;Yt4g`!tt&0;dPv(5b2UZiGYFsP`z zEHd1&*_al$eSI+~)#^jb_Z!71;3yBfYGG1JtHa&SS{vtr!6wLcvfaSV>!32+FPgMG z_N%ysqa<_LGP~8CH|$IbqEk#Q@E|)H)I7~u$%wMbrYR@0a$6md<;Q3qSYZzaJevnI z8@AFJSk#YBUTo%sR#K~C2c#D1-H@m$zvxD?R9IvyWKfnWYdJV%Hb;I)gFZVSvn;&x zTt%&76P${fGn+>)oX9nqSE#nyL9aW@@7F6Re20~L+J2%O(n_~Dx5l+PDef#iO7_>r zWSq+>ZmN;$X7_ZEa^aAw=d-5rRsD3P^W1#g&5jL!Jc|6KMI+pm*g6exr(lEJcFA^G$s z*jQESVA`+`o@zXHswgiz{_BNS~Lv*5N#4W`?Z}hesFADmE%(R4Y$vDSf6GqNT#k zI<1yy>K zjc=KwN5rLDt*w#Cph^0;HacXg;x1k&%Ty+p8}?i2Z3bIMx90cjW_R;^2`kpwBSd1K3mWG)9#LC$8~Ai+j0aO zv-F62#=@_uG8`JAbxGOsj2EECym$7{9HQZ%nx*N1*PfO4&2C?9qyDgG>u#OwnR&K5 zO$mso)KoGx zJYw88J;k7vz*BY&7v^TWR(C$1^)&O4hEY6SJD%&CQZTL5CnTSO#ZAX3w&!-MSzC}* zCfKd%2~G~@YN0%c94z-(T2FR^16``Atc5q-ntPZOyXqh_EEdzovB}c*5skGq*Q2Us zZ(1lDS$Z;I2R0E$Mth9sH3_vr=3H}9g=r1~81&9Y>aj}WuF5ArIE>&%s3JinxGO{IVjhUzv&_uNkR>%&abFy^J%X3BCUl8aa>x0zS9r+YWo z*yPLJus7GI&9vG~2gSv5^Z&Q^UCoXvO@jNpzv900&Eh54nB54R5Z>E2oB&~k5N=}r z`*IFycf0R5cVc$lx+1#I(NR@ak(EBls?m29`w>@`JOb|^!jn^+9&rZ_saR0XWs=2D zO@%*_NOJXpy)V=YvXI#xu<9VbXMn)=Nn_PZ#hjta7=UW=@_b}F4Q1miHtHpL>jNNv z0Y};&;?h+qi@g4RN*~lwDXgG<@mF!o5am4NK_-9 zhzJK>4lJ=eP{)xJq@$=Q2DT*F70|9_%u%3AtS%o#iboh{4-MplmYQLvfi7E+Q7L&) z!E6^UMcWs7nG;FEn_rA&_Z$ir?)pxoCd4Zm^%uA7K70xJ>RQ*i8B)*$JoV**8*Gt} zGp`_yc!ce44n>-UG{Uj4a6m~Rh$txpWW~sE!E%2Q%PsBI2D%K~5iUxXM=7g( zN;fJr6(WMenh}V!0M%!X|wu<$U(NJt(t%uTNXFD*Fv z!cNK@(n$o5%lTBLytghz)v4-Z?05DKqeD+Sc*tDWv5>5EXN9;N1~K0d%y-O%}fi7YFE||zt7#eJC z(Oq1gXVvz0qZmWO)(N#9w7FF4GI&iv8!QguOVZsqFkoE9ieVws>uZ6)VL2aIiJ^9WlyNNmmTJVW0Y{a7c~p?%9JKdb>Fx>~c7@s#B`LE?VHoM7!iq5-X2p6^HgE%@1`7M!1#E~ zrhoGszG|5|1IB^Q7pI$#j31ux3dvfw=WQav`|V}d@Ui20Dmi8WeGd&k$<;L==cqiE zMCip9jA9AfE6TRWE5_aQc};56P0z#(sOKcq?=sro@$zlLje~j)Pc2BJ{Q`(t(wZ@c z-RiJ*nq6I5Ms@KFJ%+=eJZNc73%MSaDFI)^WrbU2zy|4TY4)quD$l#j>x(UuJ8>_! zLkQ4O6oS&(;lgm!BYrZ7D{#4tuGM={>NMRAWoC2}jJQ~jbi!khOBPujs0x-V_gwlY zt!pa1y6WRKP~-5dT`yyJakyv)TA|$silnPRf_()h_Y3a7bXf#o!rK&5&7v#X0ydEH z!!jN_z2J|2*!p!ciQRLp-Lyt}^h|&Y0$HfldZK~Ctov-H`*>u)PY-UqFR(WDg^zCh zeYq$knB~(vx=2gD(ss_yFh!^*{dV{dg_7D*h>Q)??taGYR_FqhOaTomsd@eqOHf;rN~M^I#I^tKCyN=`0p&mjqKDTP_p~Iu zC)**z$=e=^9$!HO@lsxoTPO}W1AIxU2PVhMbIzt^B-&izQ`W-yjhyb`nVf< z4S5cuQJ**V+>%G=@r0OIpvqO0Heoy@pnFP)X~%!v;qDd6xCa{~E*G3n&&T^^q1KmI zEKJ*kJ>{ZLB_&x7Nt-iCXYg&oKG~Ko%6OI)_ITmxMWbQD;3`JNhVe{+hxm$F3^*}M zRLGSp77k|7886HUe?)ZA0{$C-o<;zNkulM70wI`@K3%-JG_FsbQNSlXYTs{2#D;VkHbI{*DRX+|OnzBMan>O(MZm)n zXxtKU77#&m?p7`ip9QWaIdVFf05#0z((vS|kC z@(q)Y+A>I$bo3<`G}i1pJ>b+$&0E{V?7Mp^vD@Tw%)`qUwnW5(AM6`V&2_zDFw(MD zL^m!3$HRRwY}_NI{Bs6y+gdkW&jg3D&I$o^5+JUo^IGEeE3I-HPvYTmEG_~1_QtQL}4d;*k~57{fLGb%9LFMfK- z&rS|TY#Acnc{~Q($wm|1N2Q;*9`1=%B9*n2XIHa!3HG!u6XD98CDg7P9D$JT5qpz< zt9W!mT5U6L1qP(Rg#g55ZZ~4mrBT=Qq1!Qigj&V{dg1LHfVCLt(_)Ci&2`ZWy$jIS ziH$4N?UoV1%$xK)!glkRjVY@?yr_sV7DeawjLu_>VwE`+2!dkqGA(sxU7^ciBMM1by@Z(kcR_E6hPCpB>7sv>vS2{&Ko#r}8>32Ty}B zgPWwuOfxYm{S{s+5h?1BV$Tl))-F%iV>nY@>Ia)2qVkU7cZj*Y%ygNwMuvoTZWBO^ z@3;z-mS{@l#Rbo*&~N?m`UT&PXTDM%1?0W-)0MwfHsnafPA#Ee3uGpuOw214jo{(& z3{ODk9ugvpcZW+-hI$#rYiQRSjNWP_zob8uthqJ)9J_vmCnWY`%?k=Dg$*$uA*3@fb9X zK?M~LXS2$)llCUd87;pqwF%V+15Hd%59P=ps~HfqP_Yx8(Ml$tF`AG$M|tttT*7!o z`QAaTj$WHLZ(M>~D&N(Z<@$zM`pfEXcoO2RisKlu?&5W|_-3B&2!wlwHIrxTGd(kpE@h1c z9L|}NM0O0r7|w#*jrQg~!RH~R3no6i-QC{?npxA`-EZN0tm&lj#ZV_FdD^Ia$@AS& z8A?$yIVd{Nn$6)UtP7aGbmGW((1W8Uxhee_6vU1PE~A;=)7hF`o;utzmq+_^23}jD z6E0J6vyy*+R7ebK2?~ISx1Ch^#$ppA0tI_Jf@)ETYCMQS zeOrRtg3Ra@nx805JmQfN**rPFsJ66Ex7SLGUz8&04^Q+V!Z0w&DO&@LwcVf<0wnnX zrtT7D{pMg)dpg97MgnmU!u8Hrc+^JUg(6Qa{#^Ta!~{MH0P}8Dz8;z3zCI30i9EdhOm%vy z6(evFSMOx-cVF_6FMdm(f2MAi;#)Om@HO-LeNKH=qik5VD_F6rdaa^tpKAZR0qui7;`u{>CH z79*^l7Z@%Um4k3-QIYLfmJ-EhS5|3j;Dc*vAqCpj{pH5qOKj4N-u0}z^4gIMK2NdX zH~21gLobBDYJ7Q_)gxyReO|?hcM4SZAQ06^5IJHqD7d@CB#2HVNv5(jCC>|XqJ&6l zUP&AvxlePXCD$JZ=+;O1D=C$0iB^^i43YUMGM9=YA|_cym{&Gq>DyT zkMY^ovChVrEn6)UIsosiSFcVSWV%KJk-!}Zp}KHrQ8LjHE9SZgexwiV))t%6CFkVq z?Xnrf@}|FDMhOlPR(T#O3wl~eI%czez7#ipN2|3HD8v~k6+SwSk$ZE;nd{Bh;+?fz z+lWxcGjY+88Fa}pKo#`i`4k0%Jn|2>%Dg~A3ECG|oJvD!BAVQ- zNjm5JzL?Rmh&o5c%~DR7;|;01?bN+XU475+b%BMYXogf#Ri9Kmn2%KtJcmK~lS4oA zBX-Z86qD@*nB!2XpjrOOQpEb8WcJPzXUltlWujbxe0L_T!xv5Tn^u)~J)`mm%Y=Gm zWd&ty)rL@IbmBPf*&@@!uIc>!(pFk2F@wth~I2V$|l)KIqms6wcVjN2* zMOE}8du%i0MHiww<6``3jkZGz4WGxby%ZT-65CbbPYVJ*Y)3i_EDDQL;N|hMs}Z3* zT?vuRtTU_U=xMb&GG&({s?I^k-`_i(5e~BJ_G>^CY<6~x;CUC-9X({0C?})NSck)H zmn0lX-*syd+U6=EH*_$7Phnzb6IV@Z!>nYl}y#5LJF zFKnN!*@o3vAi$_2%uA}LIa-z@_7CF_Wck|k8pF{*Hr!7+`tV5aaw-?Ny!YlcB9KR; zJ6PDwcQ-g#)UZ{JJ8fp9ni(;y?JQ9zQx7RBk=7mp?$GezjtGy&Q4f9?PI3a^5)Zjl z1PSK3cZel38ZIN6?Djwv!B?r?ryerd9vp|mxA9`E%Et1N;Y{MrM*&qi&_%RpOFM6m z2YpVOY&VAK<4uILDY9qg`+S#{yjR<|3AMNw!G?H~#3_E9J;~d+;sjvjVa>#yjl1b% zVJhgAK)nP60(=YMQ4V15f{-wA34FK9ut%&O_L7LkZjx?5eUHOgsG&*HUQ^XHc7F?& zopXYGoUmJnN)vKuAjQ5(cy-R61rZI#I0DP_QWL%{`DT}FC=<-W1WZu=}PZ^B5AxW0iuz|E%X~VMS^R%!mn_(tM$oMQwx&? zw>1r#Db>rp^hZjBvY&!C1+yrNDW@P+K3^yxXNO{wVL9A!a3RQTw!NX1e)xApc`+BJEmLdv@gl)&A(>$B zPtijt@c!h8!IbF3@l&>o7{Q!ZNdU{HvdvK$p0G($+~*{<6~&5QwLTcHUF;klJA=Mv zkR(JKfLpPuCQC~x3RIVkf;-J}T}yxF>_w(o(L0=1)S$1W&;kzMG1ttzrvk`UXUg_F zep+3m@{xz7Jv8ad1++bTFFNhKb7y$1Xk@Tn2b-eq#LIlhM4=iV^YNny5I0*E+Qa<2ljf%nC1V!5EuMtpbK<)G% z*zE=|mAGp&EtHpf+WVdLXVLnpP<9p-^g{3MUTqV^j6XR_>j=-DM;(6bhf|J|{GoPF zwhw8L9m$ttEijuu^6F7&k9>sb^Fz{|26dLHH-Wo;;wEKK zj~H1wqGDTTAxwk#sh#7SM@m2{h9@H-g1eh9BDBog5-=5hEGaferP#n+n!*wZvU8d* zSz)RY5N_ODutd(}6l3Y8C``x3Ew-eioDf;Iis!|~C6y)OG2z6y;hKVLRj%=L_nmQ~ zcNYHwIvMQvS7d? z(KFMTxjKXZ^e9}JQ*HVpjW?NRIY3E;Lf32Ds#aT`Ou^rz&S7KZUGyQ-=TSh#RbR8n zY%A%c*4RC|<>X>8ZDq?_Acj%YBz1N>JVkY#34C>60zm~7xfSPd#`+!gmm5YGCZJC$ zFI??%7@qC>U5CV`y~HoKGD3HGEn2I?QqNt^h{5B!%uJ!=YzecKFC^FZl5Fu>2TL=g z!D8iE7Tc!ycx^_sCh=K~f^Ok8pf|>ZIfQX?vTC(^1|l*?FhQZQE=Z1+8CsT>4V0hH zmRwg#BH1Q)yh4%%&){{7h;x%Jg{&h!?>LgH?P}L~U2^{_=B9rskQLXKgQp+yD$NMr z$HTarNhoV1`7$<4l?-HZav9)!@)bjrw}}qz@UC-CS=cq(K(^;<-N8VwQ_s&$xdMW^jhCD-+^e}(x)5_I zSisLShb9sjI|P0{9a6(ao>v{T;YfftPg2&%0+?7dP_3s1q2)C~MAw;=xCe56hp`=W z1>4|}Vcj~M)vR~uu&5%SChxw%4A@ML0rSf6jlR~vKPfD_|)Z7G9f#enY^tQOOEf(tCDf7tZ@pmAk z6ob&wl7P8tl3LuU$0o-;AYvz5#t_O+ueUe}EDtmJLuE;i0g`~yse(>-wg&*+)Ar_cz>CAj-)_TmHL%rvZKL-twK=LhgY?Si{TsJEOB zi)?)c@#65PMg${@kO1OLxz{=Kpfk*w5!rL;rgfV8j0e1f{kND}>E|NLFXiN$$|>ZP zxxFE$RZBQ5k2B&-?e4!*T-C!DM3E=JoPk7hUYCz4Eq%fE4HpA#K;3|*vJcG^Rru4lbrI;>* zhrJPoXHn0!)IcQ(3z+aP*1O;u&_cD+G4QhaiDsnhN}0sT4p@b`EdU1cAXHqyc9#*Y zaiCSXNi4D2Q+X;WE=I!k9 zrsN*N7`Oba<)+r|i^Ao1o2L`(ZKs3;633@dBE0!*uL}v@6!}n!gsPbR{A>%#ywXW> zb?0Q88p>+fz->vCRwZAA+q+1isQJ!6G#6Oh9Z><;SuTZ_#X+)^2W&V#TKo|g zQG5-F!)emDE2_xVO+OjZQ$)J$)oWL<$0TN+WEARjYn!lBbt?e(eS$~mqBAH+#fXvW z!z5EOa)Z;QsQeavsy*#Y(|Hq?=pi8owPrfjNTo;+gY|s!3vunzM+g|fE4hnZcBRn8 zy6HCDF8SLRRak0_P`SGEEksKKh9jm9t=`!jNbbCxG3Bw3_*K81rCG{!Y=A9|9Rsuj6yp7r}n?DEij#IoQ zZm7}zj+bXcsGuzgEgRqIjiaGUBH!pFG&(BERY^GE&D`;`zET94#)+(#U;Ff55?Ui)*ztl#XcNAI2LjQ zB(CLrLzurfTadX6>z@LH%A(rT_Z_ycp{}xc>_5vGxx6otWUB8wLf>^%15e3qSjoMQ zkR*d0!dSj5`EE28No$^j3(c98`Vn#aI83(kxTD z{xCE0<1>_n9(#CiqGAy?;WM;oBWcJc^wll$$XDRaxoTn1dr5zq`dgV#XR_U@e4O6) z5vydWLkOsBwWxJBD@^k7@B6_&1O>mIJQw-qM`h0X^&n^H0;hcO5o2AyoV6yy^co}P z`!Y|&-8bbx=jRE}=)P&bc?|e^rm^}FAcWqrO7it_J!ojW*MFS;JcXQDq`Z?W#{rR_ zXD1$x-`6$wSkk!7=iz?$T-qOU{`&9Z+&J4ggt2PTjX-tpFF9CaKO>-@p9LPvcK}}o z9s`tnmB0Ux$0hX*vYpQ#A=P6B`t*epv4JOIr3d<9`7e*{R`kxnBqTY0a=|3%^9fJFbyHAIEk7YRwv( zL*Fm=*tq*QENO3S{Dg?Rvwl4%@z-lkez|$?0Rg6LzlN-C6#V_>9W45dEcP}38nR*w zi?Y2-#eM;KQSJ(?{Q?p10o;zp3i~x=2l*@;djma$z;}>UJAl+*Zr%>`ULB#9zkrPX z3K3tPVTYWc+7G1ML%W~Z$Gp3~1%Ms-ef^$6_$Z#UVuajxmD|4Xw&m^S#!4+;KRfTU z9=(+!G~T@L!k|*`2E4m%xVnE1Z3kAGun9H;(>_j;K)83d>@EQNv{=Z0)cG0vE;~?g zub};Nz4P(9OTxR#=6GPIhnE%Dbfes{&+}P7Zw25lYkDnqu@Xr8!{aK;NuuRmY$!PR zf%>@cmli($Gd@lz{Wx0IJZ@I*gB71cKKQ);m&?E0Y;d7JGGcYe(AUo!mihWFw{#We zpO+J7pVuCI0HuVlS0TLQ;@kZQ<7dZ@8IOyFi!S1eF8Dv6o&Od5c5(H8=|=T$cV_E# zlw?iTj=#8B{Kw&6M}ND_`x6R3l&R3(x-S5}-pqY``zq7yDR)VgeO=<0+rCxpyKBC` z>+P1$GX8ei_m}PgXZ^YVRgd!Q>m}!37Y={F@Z;w6X79*!ul=vX-{>x7S#NjNj@fUk zxJC6d`}$mRv}VJ%nZI6d{-+!2KOX$&YtDao@OMu&YJPDc`sXx%z(04__m980WBx}R zeW8*v`^6RWf9m>I+q@J`TVx@e;u-@ z{=YEsui)n|KlHV-J{qF2?AkwdhyEAzr~dWlGPyP{+n?8xcH>WrX|rhj&rC4RU0?pK z{@*b9iNCU~{)(c1W&kD1+id*L^ndOYZ57!*=tpAwN~ubhmbYC=qKl0@JcMSfK|GvqPV}@%tlv~$+NcVk?KT+M!6D3{xA!2_q5lxM6HwgYb6Fd~I z9fcXRAy`~N6`BL#aO{b7k~90~RCD04h_7xipH5pMv=pQ$qiICbP_o|5k+AUH{(!{zq~5i7&0qKA7z@Yl|t;ut-Blgl4}{}h#WIsPm#n5dni=MTE}7|}$-__>S^GwHBU zsAMdszhj_$4*t-0QJZA{$+1pbK301WM&rlV+|ozwE}RYwe_O*3^!L?ndaLIj%%Oj_N#EPGezY3sc7HOXKDxM3{rU?k`1b2R-}0ZK|LRZQlfzH)>f6u0 z-Z%RH{?GsMyZvsz+wbZok{__PhOVzuWKjyZvsz+wbBJ7=xKTCRxbABr)Kuj0u&J;?AP}CSg3JpL2pkB!WS|&8OQ57| zBM5{oX(cVKrYJ29S97#`V`Xg)0x95SrMkV;_(K*p)WqL3OhgZ7IHvN%f{zL-kV4cW z6#dn}p}~x_G?~OYgsFA6smsW&%tpV%+ADi{H240d zM|=LyTJ7W?i#?yo29VlyAf(+{3(fyYo)*dOR+}CXaewC-m;#1|uZ7mS_~s2n{#*t; zcHJbJk?~ofyXth8!{6-3G+T=rqtnElUjjY9G!g@ZEj!L9;>$?B0#4Sb>Z0VigZw;B zdCU{a%|n)lm0wDa>a4InEm~#$(U+daxSU6&JxJO3Bs=frF z5)z?k@`-RV-D+jM_s!)&mBxh$H*0do^Tlv?<)Pzh+*mGh{|dWQSu_T1n+(f@oigg3 z9qox#%Zrvj^R&JiH=@0sSFa8jw}-MNqqDW=t+z`#o5(sNsI@_2=MpXxPSp0cwG&#-I}X-${;QloyPqwgf@c~1yH}dRpUA}n z9x0OYn#-g!<3%xNV*2w8unzMhN~9AWt*L?l2f zp2G|~*l&E%kAja0lhQZq9f_rjA;>30P_0f8v)4`(B>e~-X)Ept_Sw>fabx#3H(Hh@ zR5-=)c;2pE(&6Yu+AbMXdGUT&S7R(1*w}FY`F`aX?4b_SLt0WzfCRluIwBsHC~+~&_?dzw=u4io zn&|N(XQ~FMe$Ztu<}myT#toWJFkw3Qw=v!n=)3=)N?~fKe0%?bj5}#2CVA`h;aFh2Y z5hg)PPrK9^8WRH&Yl&2gB{nKY89pMYq<(+&7_F;*UM=+3{4W=cJb&TU@1%{cW4iUG zV=Qw9bKMo@73vkLm8zdFFxt6{&Us%@evD3v{t+E_^klVu2sJd#S7Bvv*Z4T)g2k4_ zJB2>wO$u5{+!y6ncH<=DSzm0vsE%>HWqfP=_Rrg^agLIhp`szT?4c~fCwI%&VVji= z5uSCIZ%;ZlMK_bE-cvPGQBu94N_*1%l=8jp`@Z)p@7JF6R9ouc)IfArbk=GNxKc89 z2VTBmsEQKID=XhAVjmGJoGlS8GyPnpWl^9~HAt+V$f3(GS*us8;U=Z3#Hqz;SfRd` zC#IfLmQ&aw-=gdHvvnB$*C`Et zbQvonvJ?{)WBQ(JOpVRGG-?>C9>RUMYRUMd}pL}G5fr0 z%Z`umd;Q8;vL7lxhEDNM>8}{CxG_b;+{2pi&!(JRnSTWPj1z9HFV+nnM58d7yqLcJ zRxFy&(jI&~h&Nb(gG;_oKE_VrIO4EaN9~xvVa5?{=x(1k_G&r>rL%Q8@O(06&9&2} zb9PGf&rRJGD}fS0f_zj@a#7M6)ew$Bcv{8lgx3|diDxEzZ}*DCgv27mV#J7>U7JgM zIM0o5z&Ea!vqzKrYnPK(st^H4Db_l6Aw&+Mi7AG|gGKbf1=7)m*7hTij&sxysW(Oj zmb#X94H3b5imOk0=|sW7_N0_?+oSH4yRW+&gxy3|M!NHdLR{}?uY6PtnG%_sT%UZG z?6~}MSufc%S*0{BHtniqxoDke_$N^^x`(7*NLR<=v(Z_lFx2)EHfooHS1+|{`>$io zN^$>CdAwnpwgU_8Lpqvy*j)s1d+~~L`LSWW+X`ed=||XA;#xQBTFim|x%qVhBrW(` z@Q6%Wr+$Q~V!0xzA_}Z3S+n5s&G0Z_xi~pKx!^Hx@ksG6tKjPM>fcff#*vk*`n;k{ zO`IK13b>>jep-$1z9P<-7)elOcG7i~+w>b-W^PHdR1B9!` z%kP(M`)T@)`mTJRzIp2AV$tk+8+zULboU*8S;$1W-gLc+n|@ov;z{#XoDJMY;m>}* zTz`!D!+G1txiR%z^~z==%c{NAZmD`H^~EOkL0x?7E^AWjo!K{4WpAg`=nY0rO!}81 zmsA;M61V2HeO_P84x9$>G<`h%c;&X>3U!;A-Do({$}rJ3_%mF-pkKB!aF~Bs`{VU? zv!=1GgAsvMoLO8o&*yJV&nn)^j~8tgZU#`ryzK)d(Ch7|IVTrq+u8TaOZs-eVXyU>ZgU$P{JT-<_EzXH{V04?vR712=XwY z*Un-M)ytz<<(1`g{Q+qe1_$+y#nVk!*RRq_EXoD-g?+Mons!2OVkT&RWTzT#`&w<( z?AbJJ*jL@8qRL{MANZ_Y>R)lroi_Na#NK5+M32I_+~sKYx-*y&-SC;a8NnLC>-x~0(dNzS{8{kP6_1d% zFA!5=*(+!}0m0_v{R>Ts`GY$Mgf?#VQp;KExw4R{oehVvnVpF_hr5kE&>I92aTfyK z+L${V!`*GHZJmVNMd|6|Pp#S;%kDSg{Z~pI-Y@PnQEMS40 z_up`Gb8vC~&)mRJk^5F5X*(NxM{_49V1BWuBL8~+zuNxqbN8OGpol_Cub&PN;`0XZoeg17RD9F2}EOOL5`ATDa z{OS0(>Z_KMW8FOQCtvAh72q%^1RNmo-+%LneLEkw9*2(qY5DgBRscM31onTs!C~tf zkU;3ERacJ<=KpS%;rjc3rusL~CsG2|CRTP{xAVW7#i(Kbo;mys5;~o}AWe6v#>z?K z@!y%E{edxrpvd2V48sE8MDZIDOcXr8BmIdau2UgB^*?#H2%+>-p3x}m00~0`s6J`o zX8!KRu)!_%ZfE|s?EK8{?kuqG4rnYhkrJWj#^uopVf>Gv)zrXZ*uw-kV9;Z7Fp;EB zIGwR~cy8w&-QBr7yMWZjQ{q`-tL+!l)ln|4z3KsTpB^Y275n5kPCqX}*%%0!v znjdy}4*1X2(B1XWrr_w}HQfd@?7y0CtiyowVh5E?sYcnR^^9_`z?MMMR(^Oxh!6niR&2HDE;4^e;)>H*s3w5-~*uH zK|m(VxuaqlBH;Ft$yzeth~>NGGV}POj)b<%ZNg8~sWBo*f4w00&O7%^ z`uRA{u!|g-LHmn3Wzdo7oklE5XchIegW&#@<&r`ngw_wfBv`L#xkq2_8YI=sAuO*Y@$Fo@I&Bq+^YT-Z3c8U{Wl)=E5O? zIK_8$6N0D#-s^vt^nC!Zoo{j}JRo47aF7}1<0SS7Iq z1@Qj~ZvYsNiKv%_IFL&UG$>AfMGdPsBfevj@L-kt@p6)?!Ttln@1YJAtt0Jy;Kaj5 z$=-B`V$M4hG9X^OU5nwdsW2p8&cmT5_%MRpJc%n&lZW{as=1LLb$sN0r@Q4zNdE60 zPy#pITxLlC*pDy41o?r>opF^1L@aR#My(wR*A)#%X~NvEQUflB9zn?wvPkST!k-~Y zMv`Deu(Z!&=Ur`oQD(9*r|{&eFEi6QQEKZmj0?JQ@64Vgf}Z*u{3WJ;`-}@bgpz^~ z7jj987Ln(=s(fWc~Ee{f%tBIunec_oQu+01WJGjpf38RM?SY}RJwq#J+ zV5&467r5jTYyAaG?tzg=d%$Xr1p)f^REvBP&?+D!@uH_pG8NWG?5lDufVnv85_qg0 zs*}IMox|UIqb9d2T|(77(Q3;SShJfXpjf+BuNu}+FTPu=ZZh@UWy2?g4p797G(6ns zsxp@K0hmS>l;|(L$>6JV(~!lcrm}`76`jf3>mwCkoh|rerbMQ4#=jE8bM}HVf+CRE zB|~@#h8keKKNsr+_Xb~Hv`7E+G1&Z&i7=P~p_33k`L}1E1yV2xp+_}JhSQ$4`bfr7 ztE~7Ug6a;KI|!xwxCGQE&*LTb6CfCmspHD%cOSzjXaSXVpolHd1G1&zOnyWNA+L7) zGe_kjf{h>&1(4_Nn|CBr9zoud=~tbSp!=zmA^rAa^q~U@5cI#`wPG(H6g;$35!RPx zfA7Ge#R1u3TZCeSH>Dw=kNo1{!78+)J()t#1^3;l`ku$1((~O{QeI=gB-n#c=9YCM zpYZEVW%=;~*&aj4``|l)u0(~E>dudR;KLt)d$s+`GwKB2>-ZZkGUTs0X-=hsn02DQsEW6ar=O4&%L=f z+TyDYU1G8lhdnz=m(d-Um{^5X%l#D2Ul6}!LD7D9SM%g{%@=Q6rkd(shtEiW9OWN& zuV_l`hmnkSTb)Tl3z{I{*?SYc`#FrcOF&NITMxU?j1qOKL@%9jI9-WO$K_;0sQs^} zlO#i;XL?`IHxY#=P)i2Lp4*%UlQtGZaJ*DDT3B2S{?fmAX! z;P7E}o*ovVmI=OC;oZ7|x~=jbiLxveX%hxd9m&xY=%SP?uW*NWgQYJwF~Ev$%kS#080oo)&Ne3Iq**Q)OjauMn+0I8Vo z?3dLnknk5WeupGWts50!3oZch+!8^f(EwXy3gW!|I}e4RZSP)+-(4R?S!8=KY8z2{ znu#%X26^H{8M%$qoU~gIF$8bRQv_H3*kpdj>kCAn0=`>(yQLve1u%ldS&JG~1r4sF z0NWzYUCey?*=tE^=Y75C0|t5u#o&SAF0bVI5(feVQg5 zjwr;R_uMM0{+aISq>eR2rzsPQl_VB=rK?VyU?C0%0uZ{lg1Zg_&@p=d&acKCG$dI> zN%I@&QbJ*%C}|bJm1LRgYBotmc;HwJGTsws!x!RSu4O(DmHIGXjZ6v)u@Mgsnl&s^ zZ{NN5A!p-?$)EdS{vvQ^iQoq8-jA_USbb}2%He}~}^ubKjv6*s0f_g%t7ktu1 z>w1o83Id(*cadlm;uU{|U9w5-I*hS5r?$a^n>1?GM+YAPA|AQG7}j4v;5Y)kIUG#> z+ys?BSat((IGArwYg<0P`b;FL*l>i-a5849u!m^n?NK7v9pv&GE_-tv1Uogzl8X?I zyT6;0lF7OKuz1kYI}d*euFPKULU7=s{mV4c7S1TAt+HA&zuTi6jV3jIYsElQEs8^s zdZidzxTp6*047n}8U;=w65LPZ5*_5cE`YB@Z0Vx;8GVyNiF2auY|e{woXN#nR5xwR zL!>CHtfJ2I>=9u0f{09!vxchAS@Xurax5}?Ru&!C_{SUs#fE;6x6f3av7;@g=gF`u zI!nL;<98+fIm4)Zjz=Q)fmqVas+{qpbB2dqAs%fr(bN1LQK_O`?c6epnt@oK>jYec6zpJCeO_G615dtI}Y z-`R#Ry7s1^0Q*0nFRX|9UmXpJ=X}QvsC&u+1Fb%g8k|knipq5fl~pZ)A>-aj?Fy*U z5X)IsBtn^lTZVa7C0R5PaP2Kb~#1}#{&|7%sm{wCcx)Z}ubNL41q8SzWf zzX8)iLBx#qx|pCBzIdPjwTY_Fsg;apr{fQFE&qXsV(s>z_O-h|6SZ4*5*tn*Td`ty zdL2LJ=0ABZIP0&hgvsKc=bbnyl-85>ZF@bhZsGSu5S~gDjEq4u$59nwDh*NwwACk* z3s)EFotedfB+&j)Q(X5 zr|OVSN#{eSPI18ZbUlg_c~N;rrHL;Fi8C$1m@8M$)X~~D^SH`3Pt}gpw#a&l7m^_N zAn)6X!0QHD)T2*y8@D7#C=0m}$m zzno&--zdsaA^Sc<1x0h-qKJU+#9V&z#nOmvWEC3{LhLK%reg ztfDP)xn5A56uO;<`}NuI#&BM~f|-`BD_Q z)32wmzqQA6K70K@S@30RXK`?h66NI2=(KL;*-$rV9edHRsQc7rS48@Ug7tAa7Rva0sjY)t=wAT4@P_H3jTlo_b zKjpPg@Q(cU0;)rwuQ?T|dS}PF53LwymtF1t;9TtWv)51+bvgJmAB zC|}11Zq(y4B<2a}vRSx*A_C3OIW)y?f`aCrlP~tS0(} zf|xW#)k~Gtzg;zyO^qxcF@7vm7DneQp#8Mc zx)OW@q{phkgi(w1QEZ9jRZEeqjQydVLn@Rj@2D4NYQ8;lY<&5N17neTnV&2uS`WRV z@$eZvJFMkuucc)n{c<2e+$C$0)tJGqBEC@7_b{nT5|d_m!`O!{N}k5$<=ZL_{Kc*Z zI2=}-c>a5)dRV%P(l}{_aaUg)w@#l()!i+T^3?T8EDT0UU!AR;!uA!fDO~fLkzQk_g4PzSgy_0k0j{{W)%sXd3G%cVLR*WKCV$)W z!f4;$qHiRDRyD0R38Qk6@mnv*%KJ|0IDmIcYvyo{J$LpBF>}-%^%WZtLge3yJVcO z<()8rX|R|LgY(sbXH5F)z2_Db_>N4=ONK;BG6Vz~QS25bsjv)+U2U6X1i9PPErxbA zYyt7kYbEU07N2m*vwbf&x{8akbt-@LhERfx0tKJ<2kxcD|~w6{@tCb9-DghNkW|_P`~C^PuW>Kj!yY=0+d!*BdEo zd!1F+d7rZGCl|&$-Z+sI!-oL)^^Ud*jK+K@;~%Jvk-#868~-u4ZzQs+gd*9N)b6 zkqL0ZRLKHk=Ov0kGebanzb0^U-Q{RbY#LLlf!A=#U2n;r6$`uN9ZErB-rP zEqtbo!#*O6R?l`Cj|TgGPOxgS&pE~kEQLSm#C&`;QQQ2=x_%|Kt4Qpm_{)n_k}!tyjXzy5eY`*&@ePRoD((8@W(ie(K(z$UC^xZ zI6iyvyS-WW0H$J1HSpYc|Ee8aR5d8Xs%?kx(Lu-B(u21yCnY(X?ms#~_EEbWUpurU|1C!Xnq+H*cr&OLr%> z|9SI)-N2XSdKNt5CBI(l+SEwK8`Bs&KqvEdtHDbsy{7~AmD260IXui_rqyvG)?v!j zOM*9tchBX$1vydsicEj{f3NH$x+~CF9aRQ%GKC0UJj$d?Dss5)Ikol9CP@DUq##!3 z>GdHKc;>tOXl|uJMi$exzeZih_|FSS%k!#e7BrZ~2soU*d>VYJU0`<)|AVLePOa*A zjFloZ5$|*Qhsnh`jM`B_$9p8Ty=IY~x!v$)^K}=k*LK3{6UQ7jYjy75#Y&Kk>d+U1 zOW_2zuBE{axHM8Em#?N+4?`c?Ds(iL9)2uHDqR#V!l&DINEEpi$;|2hy?Q~JYN^IK zfrX5JwcUuz#TIQIN?JL8y~>aIlE4ZcQ0RbfWQ+F%pC_DDO$l!Pt!!bMuXo0N*Gv7U z1rdEzxlli>U`KsLqB)bFc2QvB|!EC%CNlmB{h{3mzdt2~U)!TQ_6xEAk z($q<$RQSMr+jOczO1OjN`NN{wr2-4+bgY~-h7Ha{iOX{ufAJHNh9*IcfXkX4K2&oM zln=d&dpW>FLJgY*Ct0wE8HY^wdOu8@ZMXSqclmBzUYx2nN63^@Xg9@!Q7iOx&TDV! zch0SIO8oM(`qE6N5Y|-JNM`#|_;t%X1h?{9-=T}tirADZzK$^Tc;g}GMiOlEVNMZm zw5pg;Z1JPmGS-EpA0!V^UWR{0X(%TktOs{hdvP2%6w^k)w#N4?Ur3 zXcC(h8~mXEdd2|OhW|=>PCc2l1^W_k2QFVH4ZKLVyLvKPGcL{A`8!vH=Py()iMlxb zF8WRWn5WU~e=2wEmgf#)0Na{at%dP~LHP1pg~yQZJ^zp!dVf9dpu1Jm10u1$O*wfC zGH1uj`V(Am3bEKw0@<42QCp8X=BoUZ%FB5nu-|6)_ZS9PPbF6>1zRBQ2=`O}caJ9Z z-Spqr_SPT*weIZWB|5agg7TDXypv<&IA3g-RSj7OkFq=X2LwAVyB+qzcYYsTa2qc% z;8PP1#vzD0gG~Q8;LA74#B&u+ueG_4Qd)R=FL@YL5~-wfcJ_ z3mX6ZYRjzmS(@fTJ8t=Lciwc)UqR-gu$x``<5e94O&P73?iI8qc|1s(ghOD|Vd)(^)x3g_y_=Ce zupWjjx~m(>(AE;=C6R8Z4j51*r+B$S;i0U+7tox|{hmue;T;KlOd%drsEdjXtSU6o;CS}p&pp0`1}c_u z)vUYI-&b(D)y#%3(S@mdCeY8rZ0{!Nvl5)WVtE(iC+g;h)5ohup!#sdm!3K3zs~xBd z(cc+)#KLJfy%xA%?*r++f)zV)Fu_t7gZ<|&0I~8spQZmkq|Xb|wEs&~AvBurp z8PBydC$MzUvW!k=vi9}fdS>#@Rd=q$dSnbTY^POZ_+w2-P74o(*ih&2rPlIpB{t;Z zmp*J?pla}0AQnuvmEX&iZx!ef?C~^Gad@J6?>|V}Fm5aI)f5*B;??VzBYB5Js-~=p zvPS1T(|>L3OGQ4B8MqA}C^k;vzRJ%_R>x(=hTyiP35Yo`At;B=4OZ+YSc<{)qsykcRxE5Fm5*kJoPZ_E`O6 zk@K8}%OjC)#S}lcqm@W<$)WJ8_jkt5w5@=r2(%w2 z$cY+o5w?l4c)zCIIL@${BAFr6nSKvqy@77^gUod_*ICR+9(3;cqvI<`=jqxtx%gz9 zs_aHbmiUdVu2Nn{5+pzT1JCR?G`J#-Z?9ZC*!9&4oM&d=h7NSMS1 z1xQ$W>Q#ZN)pJh5Wgrq4?4!44f_4}FB7N$>Vm<0%Jw4PHP9|9zgxe)(7vCUrE^dBE zTZt_sqh~;!m=>{h_CV;1u?Y&##_Lfj`P@UkCNWyyy zpRR{Y)9~u=WF)0UW&(Y*E}EUe%U0P$sUNk!w8bZ)+=GnQPZK^oa4(*wJkhrLm@>*+ zp@MFa)nLA?_el29Jz@zdQ~xCUxCHrfj0G$C;65KZc7eIRG4$H0k@rZsJ)gGyv7@;S z;CNkt_xRAPA1#HU%UAGb%3cJgGj2~o{CpuO;`7L3hr4)&KH9=Q(5wXGQWn(dB z!}i3M%yY%_=p_5RLPBqp00*lLi5Nuc1>&H?qCB@V*aCtO@89r|00Cj7y5^`lBeKOe za{YkrFQ1Bf+VVl?Vr!baH?_mU&Q}QjtTgorXf?PiyG}SRr7e=X6?c_Yk*bapZfP*vzJC&!kQof#GJuyVQzz=C{ zVLZfj;alp7_K(yxk4cn36d>2Hzt8(#EdgOL2gi@?Z`Cdx>6`+Cy!J{Ndp>(!8^m6X zii0mR#G!%|6^E_p1V|h>Bz_RHzX+-FjxvkbIPQc&Ejvl8*yzB;*;a^z9|cx zw)6#=8%nf%PaFQgXM(S50?t3msvN{8ch-aQJ}Elj_vM6iSCHMo03@4! z#LVzPjEMvm0A;NFeEkIO?{|%&`#tIw(}vh71~RfKn{H+1_^E1&uyFdR5d zS+L4d4>wI!&!tC0ridbF3*kqEAMtRCieD%5)!*gfg3?uX<3*1}k)CspdMwnnxUaa9 z0n<7)cWSynV$gIy|GB?I21$N_r8KNjWZ)|Vme}!iL*NsBgmeYr-I#_dskPVBn`6L} zE&j>%6v3jXa-`x3Um;L#BiARe-M#^A^YPPiTMPGeAGS#V@Oqzdu|2)|jxn;g0jN!A zkt@eANIm{c;TRH?-}#km`S`agCr8WmAv_>PQW$>1-+T}gSdrkpr91x&8`W=kd+2}E zNo8pO&)R&4*4M6E{)rX@*Wk$5QPBGoUMHNpdf!8rmcB?rG+M;jX% zhoDmRF$KJ`3Ll6ihcs+a5xh3o#}{$uVDu@+Xn$f)=!|{lQTl1Vd6cVE`lbJ{#I8Vh z*ghc>0*QB^g#oNJ0VBs-p}(rH=+Rf0Es%es-l}6wnNZ3Y_+G3m2aSL2q5Yr<4YO5^ zkG_~=yPw1Q|ifTz2y1Bl)jRU_xjxjsX0Os zS1(s%$iKN?!q$6@U>+s1@4=p@z?>=(+6u18Epv^%k!UN;PZ`~b6 zWXV4cnH=sc4Db9ZUetG+tr0jLmQM23GfR+79F)$S4{Lq%!A7%Ea)ZWgW`y_Z!wBQL2uIWLC}L-zC%xKIhEOG9f=rIYO6SMmVw^(9b-3Oiak`FTyIq3j`Vc3ia$c<`b$| zs3KdSV0VqySVj=*WWqt%Ia%C)Y}KNJc5uWcL?C6E865xpe7wTD+>FjjRT8;@yYXlW?<+El{G&0q^#DEbCucuzl5}Bhj1x7Hq#^HHo)_Te0C3}z z0MD4G6AFgCmP_W3O0dDsno8_7FX`31P01gO{5mWWzbO_#Me~mGyQ;q>633$L2}byb z3J!L#$lS&in0Co?FG**I_mh5vJTnQIw>=O{NM1Gzc1rCNsxGDVBG}RmZ1mKfK_l=Kfqp#tX zorquZ3lU7&3MCa8uiLX<&N8ct-#AmPo^Ki6>D1g>VU2CdU-fnnNi5ZnJ-Zc>vCb<`R|&wzYfCozw${XKPpj) z<4>~4hXkw=9KWmS`A?2{)o}YnA)&aL6-^1$^Qfhx7QqV*LL#P(Ahwl7TY6Oy-tcDQ z*ds?Oj;oEKgWt}OG4pd^A?tit2r(6t&nc=V%!^5&FPIDw=bsK*w7k<_pB=>cHEx`Z z{lp~za)~EvR{&$nY5i0atl#%=b`X;ssE^GTj?fvzJFH6|)8UIQMwv!M@s@uBGLhiT zm&U8uFJ#@OZE8R2j{??zy}F?WD=msmPQt(GbGnxskY{|fp_NmZ8~yE^KY(Zcl#3=e zn4|Vk@f7J#yLFV42_ll)SS&cMzAc8F)N{Aj*^+Td9b>JT4|ZhYSc!tiLQe90J9_fg z_OoumXvpZnpeaQtphihWM6+=O;toNx<9;&!nln|`wjS+d{_+RSENOaN<9dD+**z@0 zemWvKwcnEN*N5};GRvl8IALA#_;G_84~0+xwu_%)uR6d59pstcj6F)zB2@e;FhEm{>~Dm|cqbac9p8M2$W?MUN-EV%O{kC7Ivzr{gd6ao}go zK&hsv%wbUEXrP>Z&7GBECpeC?Qv2feYR_;yYh96AMonK}Znr7rj9A7#dj0ZE^xiaA z3@Jkx*407~fy#X7$__)J541960GqW`t&-Wa?E$&_SqxnN zn3r(!%|iID9`KN7)Hml%PwVoop8_=wF`KsU!%{e}ipF!9#$%dEYHQ7!ylQ|_^k?DZ z0-5;5cyEQ}``lm}S!uB_D7+^UTGDAK9|by6%=TE#SOOp#OI?fhP5ag0Lzf3oeU)9L zw>8Og?+lpK@p+sK7j(h$o!IT&75c-G70JPDq4jUIK$xa<+*y)F%u?&rQr?E~Vf4X* zgfe!iVNr-!5mg;bVW%SWiiTf&(FxZUYbX%Ud)a;KBtN~^`7U8 zVK{Ttx(ILW1pwY=t-~-=A!0^*shHB2iyTH6d6Q6xVMlY}V}Yk4-$#jj2xzqrhTWtS zOR@RHONGUZWSH}$)4h{i_N)5Er~3{6(yI1Na?^+i+|OG$1FulJmOs+K>36~O#itYV zH2qYz?oEwjPO`R2?1?xxSli{-FP~l>6fZk|JOvm<b1o@_euVeygHI%m(+lt{S1_HFOO!T-iJ!?mcIM@YA^s1o8byrEhVC=Ngz523I~} z)PdIEI}SK)?AvAF3Vlhu0uHE%@LQn?S*4#QUVOzEzs;5^Lg$R-&zRyY#&F=Q_uqbJzq z7qi2+k3NE)Z(bm3m|XMY*uL{GKaFs9;L zhj(_Qe9pIWr~MA143Q-Z_X(&$^GMaYx?!obAATIdA_y8AJ>pNoB1H1Ru8yU$cFwb8 z%4tXxhXmtcbf_~XIG(RrueGP2|BqP`H+f`(xvT&g9=kViBEyX`a8niG&sgX%R;OxT zM;I|za=(6MHjIFbC7su>1q4@ax~D~>7oy|4aYTvr8Uj|SNXs8D?Mvsw=ek=VQm;CbG5o<0rg8JADQPrhybOs z(c1vLH#B{^;jILI=N(H+QJfsRO=ojv#t(meTyInXSe(_8Pdf3<(Fq44=z21x5rli( zsp9Mbf@nyCe#7$0d&Y57tV9OvEyJ?*=(Ap&rxTLS#mJ$wcM)(hslHsCX9BB!h8I6y zN4|Qk?=m*zB1+K$R8qU5c?Z)CK-yoaMn$2Ojvw%@h5*d@*qrK^Lr9<&J5-FOxj7Ju zA2*a+#r*BtQ&?B85Z?OskzxB|1krZXQ>ip&Ba`s|{(fVYsyDD6wVN$D=HXi;QC$}#PY;@PK$a@{u zeQu~k6edPz!2@wL50%nzxZv{w0+&ys=+f$Dxh{(P_mJ-2z@^Tgjm>fq@T*re9?J>E zP9`f~0q~H9phqAA4k2VnCYDgh0WMMU3s@_M6}5VF>Z1jvSn|Vk3lM#{YS=F&qGsbb zk2mh2B4I;chhRhN09dSVT{Et^5i+#(q(go$$9Yt-%XQi&t$Z?CT5+QR>RhGDtkGrIv zlJttMNIE{aRseg%ik#LTQj{Px^IRtNYqj1Me|#T0_MCqIp==l>%LS3i+DJH5ob@xl6AA2#(wCs4l%jz89cY=qmpnw`7qNP@#kLIn8d2z#+|haQLx z;S7D@)D%%|c3(VdTp3#E0dgzn=phfbquNgWK7V0s5Y zxsxR4a;yl0^fUmImN`~v^YgJ>2El{1tc1?^qeQU?I&7>cxLF)m7LBoxK=40W>opEq zpONTmH;c9T1_AWz`Oom4_D-pRt3^lD+ap2HFx`)uU6XXKl3@A-5<3l(-|B}H_)N|#cQ|$94_6y&%3Lw-Ik!i6fU+v z7o)ht)<3!M0H7@43lN4Q0op}famm;w^9v*;I+)qD6aqaGz9*6u^{r@~*#oXah!TbL z)4C4tC`t+hy-kTf-uPV~a!+!fEi32kFqA)_C~^aP7`udT!x&jE1^ z+eeyWnTnQMPX{2lCk>-2JVaR@1lBlrTxnoZUJD25!?cRm83u&$B-zmA1gO$B*MuUTByeJ*(DGk zTS{Tgv?oZ_!aQuk2PzW*4%odXj#d@A!18;lLQxke2IVNaA*HR{d@ozRbuno(fTRCM&tQKTtS)W-;-0Ab+5XsU*Iidg0N{L(mkJo`cdla;5ZsGR?WeKG)tP15a)X_*0i$em zsYgx67Iv+8U5zw)fW)VVUN|vRNUd{`JbMuKqEA06YToJ3JP^?SD~J*H|6*$YoTdrw z@bad$Pmm>h3}8)m)k{PP7E!SMKT>Ch+p9~0-Z}Ews_j4(cf0CY`eRY?t3N}dlTznA zvg{Xt6MD(ke7;npBCD51FEMM9UVnFc^_%}QgMYll)gcGFr+gHtj6LMsM#Lj1UtJLc z2uS}d0bG$8Xnp5~t+df@{hEq&2TVBA3?@z>4FalWxBEM0+s2=JmzQB1$0bN~B^%AX`6}vic)))p1B4zK)J+%1i6mouMCB_;K22; zMdcwj;DPr9<-a9wzyEnt@+eJV^f@&5d;<#Hnww*W0GLEs08!fvk)?%c(b)5|2^(`` zisp$oTi&S&{BCBkf~%H)a7djb5CIAUE6=%(w+V(GwjTl9q9etyoSaA7ypa_lY9_rN z)IynE4`9ulz6Vi{Regc-Le^4-MfPK<+SiDU3{b-jU{IB90&m|p3+;8_O&fTwcI#*j z10wn>btHiPR_^BjgqY(!9lInKv;T*ww~mUcd!YVlh7g8s7!U*mq?K-@lvEhHRWRu8 zZWtPrPLW2Wq`Ra`kZuI&uJ_FI{r%SaF4pq-=UKz$&N=s-v-fB3OQR$bisICbD+9^O zBV2i#VrmNTSwCl{E4s$0jJgvfD6(X4n!5l_Ev)j=XywgUJJy>+-@7Q#ke2ZZppiH> z(j11&F4vQ}-=sTDiA9il?^gDNgA;Cn$wjKE*9N4ts{iq1qhcqN0SyOk;)kgT^210Q z-E_w@j3r5N8Ei#446niNfve!N^)~?l_&gae^IWIxQ9QotQ9}k82Dt#f5$<#g0Dm~mg z>>!jB^AfiRBGXp9tR%r8ZV+!9(UP2?at}yGSNPY?B;-kkMMO)V`7P&=%O7ifdmvgjiv+p*FWHQ)86$?fldx=N9bP`{%k+;8yK=^4nIIXfy za&e;|Z5m=Ci^>ss3PZW76@cG!gW>bV0!Nu>|DG0X&a6yh?N0Nqx7B>~08Y@Nbp83T zzMZ4Iajb=PfVI=*vlWoKPuF^qZDDl~@{-J(HP&tU4Zypo*xjD3I)oZMkqpTQK-mf9 zFHxi2gM_F`Zs)~o>Z8344*KmX;pnmdgVRxT?kDd@ye3n+)$0}Y;`x(>^rz*W=x-OE zaxdD&#;*-ySp+a;@I3gKr2A$&)x+{%pmJ5xaC@@oej|VW@^`oTim^}KRBvBIE1fqz z`GZG?mc*#Yur}=G3-T=`_`*2V=xx|7h;E`Kl?QQB8&hV#;xUfIxPX-sR_AyB=*tJq z`LF4WVv3*t+vbAx?fp(&>V9fDN)V4s8EF`-qaEzRJ$X&r1Le?w`MTln+&p6zk#B!c zsu_%~t6tSXAWE8fIrvtSe2{VcO3m%Bk_*^c}ci9~uNA|gADigh#&b||49Z&*fm60-LR=EA6M7*DpOE9a9Kon6E?)WEy9D zlr#LtMySn9i}%Hq$aJPjhuQHOTQKNT$S(gHH%|3^jpY4*wRkPpb9W6#o3E6RocqdL2K}MJ;Bg?bsY5duqwdz4hb#bnmcJNcwg_mYIYx+2PuI` z#;*YcB5bizM{P`s`Z>FZNcZvqpC6E_d7^Vd0!9WHN*V5O>2S&EFX7b^^e`BAU1QqP z#TfJsncPWNvu>W>4f(_X7u|Bs(7pa}BG&%Bkme%hP6C%nNal9#-N{Gz1zS`jKMiAX znW)#zVGl(Qt;36}PQ6ka7V|>`;J(IT1`EG@G0TldCq`3FZ`+Exu4U*mXbZ(f8)GG! zApSb(tNKu6zVbB-Hx+)N*+)}K3YDxW2?ys4`W~8()v=>}o4@GWzIEUC5@sI{7Y&|#x64?vKHkWr~EAMeWN-ny;sdF5f`(heQHOBM(vgzh%;5Ae4pI-O! zh<|t!FspT1ty*N)zS1P@zuwA!;Q(8=^;aEZPGgFnnNrd*nj(|2x)P;uOsd~2VJpp^Q=Nq92v z3%M^jH&lfVPMUku3XKQ5X0ydh;ogm@aK9eNBzd}mJbF2P0wS%NwqC7&-*~LAJZV1? z0`(^-;v4j*7YM8WAikLz z2Bxs3xxuDKTh3QJI4!p|66dm+^hewPqnH{cX7;;+$J#bZk1e^sbnT@oFrOn}Elu9{ zV>CZ=m32FE)+HBF*|0x78oF*gbYO6#6cWJ~CGx~EyDR zH)rGJxJDDLY_Yw=-i1^Ep!!k71roAxY+i7TQ)_;&B9TDIQDCfN8bMqD7x>H&v~}La zSj4n*amb#d9c@Dr_)+o7>CO0|#UFHSso+t{_7$$Wq!FdcmDrCw z6?tHjfhNi!sJs4qKqiFo^84}D`t^1}6@z^*V;F|!BN4Wmb(5e^|6F6il5WOyxDmLu z`1t?t0-zbePv@(){@F3xgGMOgz!}gBEV9@$^1xTTL4HN}ALy0`9vth>6RM)g&*@a- zmbKhnv4>aVvmY`Q=sZGia)bkO#ioUTvOv?R#yKFR*SZ8b_igR{8Hnku8-a z)05t!pV>QDu66S_g}A38{YOQnuVe^{g*5!5MeoTyvG=>b`_5C`rYKKMzy6@;66qW^ zAC2&5 z`-tIm`Vq7dn2XKp7w!kWU8|YR!8)?o)-0CcC)=KVqDwEbv&qa5vPj!$bX zNfrRpjTOpd`IlhB#gwT2?&|M2&1r?tzu0&-yx@>6W~gYCz1Ml~p5trYwy}E8 zQ}Reg!;;_+bjLvTE1)1cUl=&{-Qx8etl;VBC}~~K&;KyrtqdcK}_M>!|H8QWY`zR=HR`6gr$M@Gf^VEykdtb4U_>8rTRkCcD$GvxpD2t0pVl$BL zQ84FHH8f{@6en$8VP5r%#Wig+M(0!cIa?u3KuVGvG=-(|dud6C;Xe{i-Lgm!a-i5?2(wrJ` zNT`u4Z-9W->%Z}zew4L7pZN6L@ULgC9aK3uNFi={EV0xDo_$j?uYO68yr_O3cF_$z zpbEw(c1}sb^g&4U!2%IH7Cgp53p10mUmAWlq{}mHd6KJ{+V$j!zgiqw5G}p8-a0#q zFt}XIo|BYf4qd{APmtvtD&lHTHsx@psLAWv!IiCX78U4}Q%;7pdEs&?y9<9E(CgKL zF*V2?@tsL!=Y2Y{(H=6kqVZ{eU}~zF?^Zm2NB(B?%T20K@fF5i_4lT>H(t$@CXd5f?B+!T{;Br(ib@a7|wY|1c}i$g)8=^PP}E) zfwvv5?=2u=`!M%CiJZpy0F;j3o?#fvDq@WDA->#Rz(iDp$>v;urUv&mdC7Sf3UrkT z-1#BA2Y8t<-%G?*D~HRbM%moY?}76=-(dc^UrEByzbNs?iszj}UBR+C?w@H@kXmq# z%fRwa8RMM_rz;MK?Hw_iwL_&sB5-o&uDw1zI@a^hnAodqZ@!z2c-?Vo{`%#%K1+C} zu>a+ox=A^rEo%&TV!t|KF+Ayy)>d}Ap_i;i9SC{9gZP2jJ}8S)iJwD};sHQXMxmdN z?6%Iji3L$o`Rc7^qG^hnuveJ%hvCH1OvaAUX4~PaQp0PodzjJ;;p7hSzEC zxiWKb29{46#$=)Smyn8I*w;;mR4k&Xo}yU&*5WQ&_18t5p05MBLn{t12l$V0KIEV@2yolTi6elq|%Cc$sGpcvXGVu`qHn0>x< zicQ?nN3CWK>TB{SlsCy~&0r@meh2p));@YGuHWKJoUtF5b1u(&(Me;x2I8KqSbz2> zb&L&{o`4WqHKV(j4f;}802a|f>ZT2GEBJnXcaU%_1`*6Jz$)lo&^LHFE``Y3QvR_g zMv!2HKtoqm=yvtjC@_Qjo~SzEpaJ0bU{)A(gL5RR@^xV}?H3wF3HVt{3 zh;2M`7LOwV!-1h@2sFvSzE8wHf;s&;O7YkW{$B}-u+vn1^j%Um^95bUhQV7~ucrr= zY$-O#O!A-Y>w-a`4C=ivePsiGM<9MOVg}?)wL)0<2mhh-U^@t%ka4?o^Fa`I-txMm z`sFhE$?iSQ_L@hRr%uX)tklO@VmAjJ+dn%vso{Xm9)A{f$+>#oa)0a3eQ+RdG23HZ zT)JVb>3$cIKW5`*4Af=Jj@O>W9%D+ZFBcUz^m_>0TELfpnLZ6JMr%-T+F<*;(Io!E z5qk{lZD*qs#=wbbBnW0CCR1D6`1rHEACOoBc)<*)D`K{17=uxDDp*N92JJ19q(~em zq>oG02^|fA5==i0H;>v;Z#h*Gr|BlN;zk3hJkLWAO_2saSM65)J|5qJRw7(7GmV32-`5ExqL_*{% zlGKeMnJBVH>{JN3d~LDLNKDqWRhn@i%_}jv--JA^1QO`QxplLrA)0^R<1v;bI8*ww z{Tuf*o`rpUk3r$!n);`KBz{mc%g+E^DZV830g=+>plVdqOo!X5Pk@5+pn065lnepa z3IoocFP$=4ESoL9X{$8GHcXuypD`T>IuF;#3q_+^*$dM&D-;98+hJr=mR!W|=3S6u z@)o$DKU{&UIbM3SH<8_GcF;R%%zA>^5#I`!>Nn)+4r%aJ7O$uy$Pkp6YQ7{#dt>7sis17a7nLPd)!Xh(3C$pQauPBAoTr6t&LSl z-PAQQCICN&r-C-nksIxU5}Ix%h{chF)zY(^PjH*w{%eXq)Zq(uO-y~P#RXPAXkwu7 zF2`@MS_febZVMPyo%KG9dDVi~45V&CO6Q>#CFS`=yB@kO$@|OUSAk&k^XszxkXrx_ zmX!n(>yI0vGxxxz0t-iAGpHUzh0lRPzRJ?ag&=Sm23&LoLCXWkS9CBbPjQX zS|2b8p?Nn8Dnnnrb`9td9v%GmgieXmbV>7#53}A$W6kMH zBZJPT{_t~q)8+**_q1p|b}QLGH|W|losRhgnuP z>PJgIDsOnWH8d4ybY141u=7r7KL%6a1~FO$azChyM>$S}MfI9Ra3z5%9`5sAMYMLr z>A0F?dVA9UZZu?mLDO~av)k5LWgDGb1;GtCun611@P#cP;i^l(6aD(IM{x`XW5zzP z3jSL&ds$)LcTaU>ZV)tBwCXRRB%cT#!{;PJ5o|GOZmr z&9x2N)!ILkjojJuR~6nfn6Au+2g|b#(k1MgrklSLO7_@qdPY~(o}%`GsRX|9IaH2d zARl8~Xy}W}r#6A;V@?4aD+aV3M=vK-5yM+(LzCAj?<_~qCk8J2`zvFx`WfZ5hGXMa#dLYjB;dgq(J<7Q+*BHJjA`t3I%(|(w~Ux1=;QJLzutql1D`*y35Cj9 ze&%0Sd5noh!qRt3JiK5W0lH-9AbcCwst^*$#^M*`?W&rdr~LklP92 zVgg70Mhq;6{V8LgTWj}Vv-3rt`){wgSoPt=!yI3Z(0~;ja1X`;Lb*#EAlX+4m&G5epSEb$h;;*?h$dn8kvRrtsc2BTvUa;BroV z{SWbvf)GH97oJ@7jn7BVFeHy_1X8tC^A;d;jg@(i7_U%@8I3b+X}-kXld(4tfz@*! z&T*OyHiNhD2g!V)OSW4Uo6P>64dFv;03muwj`}pGi#Ly%N(^YWag$4*`;~hTLrhZQ zqrc5RE^`ngB zNkmjDl$MM*fjmpq5M?YzKRv8ehF*Hhe_l`JYKZ1#Fp%iF8B*;gxhSI8>1k*m2Dq}M zq6r=ca}@zb0~Fj%Lpb|GQg+8{2#TPq2-sEo7V*$N_rhF`4kmLMqg8A6gQ~R4)>XB&C1A`}V(Zr~s z=L6g=2FR5Y9g<~XcvL2+{@(Xgw11#`5qwmZ0@1;5!uf@HGO#uZo&xwx+CYQy#{w-Q zMZKUpUvDxd5b9Gc$QnVF5Ot#-vkfWg(+3jEfh4#Ry4~F^69G=B!qe2D@)X47;D4Xl zI%V=gmPHX_$ke*HhzEbNF}wu{5U4&*fK7B9PH2{e`cd{U1$2fodDy^omcZ%(+J}uM z;8BoyN(XW)y-WN-FBDuNi{tFu`!KimM>=tF^=f=xd_0`Z3vzzaz)k``$lh#|j6 z4h2E91R1&^(-aj`Ri3}m0IL(l9mTn<_BQL8gxe#EXyFehqaP}2n}aJJ6= z72l8Sm6pf*EPiJyZ5h01kVzG&;1_S-48O4;viq;m@TEjx0Jq*jtjEno*}LOej1(IX z(CPy06&$I7Tlo=WMvWg*38y7Vm-<`s%`_6rt0GTT5y;~BQd2k6c6nZcVI!=Pshv=(UceuQl%7qM!eO^2pfOTK`8%Xn=V zXk6+Lh^z;kS(K#htgAxHMHe^jA7D7@4g&wbZ>JD&W1H&YPOCRkJ42?+PsyJ>0Y!Ic zhSA7d^EC@BFU27Ml5ExEL(TRzk%Ca+zS$xoR-`nR2_4pFiJ(IIgl51|;6e&u)r<3# z;iI)a;vkM04&*Z4kJREI6!1tTOMBWg8YYLh@`G4ocIqsjbNSk>f_tP+0w$NBsg=IP z)>(~o`V&$)TYT4C&T1@~129)14xmNKwGRf0mS7U1`Ntnu5LZM0FJAs4U9Z78W zRY(AYk9pTqN---!4(9WO6z9FkEnJ~rDr*rrQtGL}c=p~kx1XToO;(rUb0n4tVP;{w z7&MHpuJ|pm*d8<}g(bTqyEGz8$n|u_B)|+zT<+$Rd~OyoMxT|(3kYcdE+O0xUotRI zAP0?~6c-rI(=z!#>;SzPv9Dg=V-}l@u;N*G*2`CaK=~dp`uyUVQpmc#2w;c{?Rh;L z7CRPjUB6Sl%VoIzYLa~ouvdh((N7o#LtK8E)+(~L!@L!7jbWbjQ9yphwc7sN`^dzS#WswD=FMcbnFgCFtO|Ld9J zg4Bb@U&N%gerXEO6N~jbUWjbxg+Vt!D^XU*(dCU4B)dc0?>!P3L5qSR{vM6&MZumq z&W`NKnvW0hn-YB>o_&lHiWP`Hx>IPWp6QBX;(d9!@ufTk9E>AWp48{1fGVvBFhj~= zWtr(v-Z`UY{w>y$YQetuB?^+JHdt%&qM1B(qiNGQ$js(bYzf`UBO zh{q!@pV%YIVAh-5fver0Xq#dIk1k^|S8ZDKAulf707 z((Lj z&xy_r1wd+`K4bym9p+x7SpHqOy!4q4owD|>0Ty46k=0Yt08cv+Hw;$ki8#0vuNhwE zw*#QWhgpkjWE;)ok&ZSoR63m_kdAmr33j1W(SxUL$ABETZeX$`blU!VmNPeMm_2v@ z<2KoyPSVlide#lvMs6CVDfNl~#uj zZa-F0dV|i8;^y?YvgAPSVq|et7g*B`4FFPZ1#tyueI<^s4o8LdCK{opLos=5_GunC z$!y3$4Ni(%WEd^9Lr&4|X?59-qMMYrDfN70Tc^{;+%tkqjviZQ&b4=huQx<)-rvVL z8`rCqVK)k$LoFDjIZf^_dL-k_H98mX@6H8wufYMZaCk@>u>cH2(nmu)5*~FbKom7%Bh3et3c>CD02W*vRTZ(eT4NUh}$!J#v=Y@IIzh%V;_@t z0Rcr1xgd6dI-U98pVqs?CapQipg5#n2#~kL%U^&xfJbXVD3h#z!R#L=8O`WTEd5Uh z8>cgy*7cZxcm_T%w*$mPN;aSuJ!w=|if@OMhU|bO zl^?|7SwJ8Y`lX03?T?aa_5kqGlX@5NxEm4VK7$~SjY(i1GG<~Lk|Y)F(5?UnhmJH1 z#62d?uEk#W4{#SvnCFL7Bt_^#GKRT)^)MOz-QYQ)Lq5Pe#&u*}LvTOly&8{d@B+_^ zm-?C6b6%#yPNEI^^JU1jd*Syc*rm}N>X4Q^})z*rIqeoMW$XA z)$1=j6ERktK$TY^cY=wq0Z`N;@{{x)*B2Fh{&tgGKreYVV~%Nz0&cxpmOq7W2uXD7+ zd`8Q``_p(@Yn)DnX@><{?`U?x-I6TnEUkuHAlIr~(HRb7-H*TEf63(Qi7$N{75xaP z!<(w}|J-iC{@_C;dO1LvOvx{Z~tI7H)LSNUT^dHAddEs z6#NcYJh!}@noSEZelsk|!cx_c#&oTGEk8rLxFJ=+hSpbVvYEe$lS5$Gz8@Qp6UAX@ z&`t)tE8q?l-^udLQ=5Rb7*G5!JnsVGjmMmN>2n~LT! zC+;?AZe(-)0@jH+Vz+=^Y0k#q_XbR`U{S;+7cEVn&Jm70Zlu65As zHWMi?Fne8A+Oc7_e~6N8CKmPV(7QgVubGncT9HBiInXU;G^mUtXdjP>p7XiCjrW$D zzFq;iR(ecx(18-l4-{%O9~3IFHH2(9Qi6cRj=7pHjr# zk-XOlHnC)1cIR-icl6}-l1xkMRy6m8<$R_wfBP>5ShL_C&li9DV}qmpmmWZ>Ff|5f zBvYY<`3P)W`L;*n_}`yC7PHv>6*906ns{F1LHQ-MB^IBbvXiVmS!}?3zX*`6FB@eh zx2t|br7mJKZfR>V0um<|v<|BGpZjh3j*Tmrf{}Y07*7WDqEF3RQ_SeHkyf@y)vB$K zP>Mn3pS{;tr$~VU1%_+B!4n1u_2%cIIYu9$g2PDW-M0gOq^{VnAQb1zo6ph0GQFf0 z2yHW0T5pNY=vP^M@~GbpUBdi-wb{L@pXv3gWAtW2gD{!)5zFERGy2`xyrST`4O`a# z1CF6aBK#+e{kveV_cGicd#+ZJ!8tl?S_#+df+hH|h*w#ik|oPYAPz}zvuoOad$BBR zB3}s8mIuad=s#KRosq=AuxI$ZW50DquuY=OtqJl$y<;Vyqp?|a zKzK7vi^WG`_e$&}@X#(!nv9N6#7^wwPds5u2WxtB?Hi-aWjYM89hQB#mlb-)Ep`c| zD)GRZBsWgCNv$*cJI+UD1jI5PZSAe30SGA*F{Z2xt)AlJ(V#6wqpProFKA8EOf%f* zMAiYr#o-J#?$I(qf^lytiML#qe&FZ)bt{{^U%)$;qFj_{ZLTmf{5|@G{w)Ov%dwF4 zclgFFF)l5e+(GPUk6WP`NZecdjgw|dZM&FkFe+u)nV*XhL_!tQw_H;zb=kpqA+x_S z{(N%xz3grq)+IAheKtUbcJICnJpY4_#up1N}qrZhuqat8E$ypaT^_t zJb<{XXKpH(`(6<@*-#p$#y@gm9VX%qY7}x-PATh5b27qU$95+yhBYiU`&Kz{$4Po# zqYN?r(NcU1g{+ANc>F59GO-=Q7LQrwU`kIyuEpDa8^2{ z^)^26tJfzos9EYi=wO7AGw?30=TCCe0EypwXQnkf<9#J={AR#O9WsZz@+i*o#`E}> zPLz#;2^)UX8Hn$}OB!~*z2-B&QEWaxcD2+~l8nGZYx8wR!)ces9VMwYJbCD@=@nKc z-||i2aI&e~vPQ z2)UGmIjc76ESa2#@i$5shjwZ4kepuI$3kS7{b-n>#9s@KcTbw0tzO{=Rb8oi!k`3< zo0t3(Xu`gbC2Oawk_!n?7(3?H{92hc9WO7Oo7k0WjHjH9qJRMr>|lVcqLzU*_MCRT zKx{2vKEUCw+;i`S+AXCqm7^<(vB*R&|?v^f$O3>Uvr5ic1aZIYPGCQQ;aih zzC4`Jd1&87rXvvN`KQ2gi=xM&Fk?6jCZbVHE%7mVl)|$YG!PjJ=$YgrVxf8?CZmUYc(qpJ3`kmUH{)bBWO0a>>`(>pxa7GD7 zznpq(SyGq_)vwWAp{_shIg>*2$46VAO?N)l{e9q)VXF0X29x1$^OXFHp2r#)vDp~* zw@3u=W5<1msn9DbnlPI;;sW;bpOgk}kCoKPb+WjNJm~f??E^=;N5bNs8qF?MWuL&Bzn4P|_dT&kT7a}oNQ0Q`&iS^Vg;G4*C=EWbsV4L1k0u1abK znLf2X0hNrJ4``NrpmOWr@viFN1P-FyxtjDJPQBAFJirfN0k+>iQL$9W|uxae?C z!=!qNc3v&h=0zoj$F!&sFz(+>->23jsfiXG&w_LGm=&jY0*TM)3%PT`&CYvF| zS=Rp_PLH$$Z9&MG>#PJWUs=u9@{RKcXIFtAzDZF&zEJ<5AdwM9qq9UZVFbrEn`qKj z)kHf&AR+ZWl~rQ8*Rt*ip_(?cmf`V;u-`QGS22Cs%ejc=H39}Y?kCk$M*X$~B*omL zsm&j9^(QMY>ONKCR_((+R({x9t)rceaC(*jSfL6PYl_>1LSwpX+-diV0wGXRV( z87tQ7XAu*vj}|JUsWFKa3xwgYc?AVj^(?&cWK3mT z@J0MJg0sXC@l-LfeG{XSb|Y`h!7{VqdEJwbbbQ-QM1*1UAUK7iU7F*Fc!+c9`yf+$ z5VNWS%IS~k1lFjR4kQDCpG{R}c>>7?7#+APci9sXqa2^jefVyX>gW3Y$GU-Ts6Hr< z8NV$Bs;1S}v;g9r?eh$P^~)*Omobkyc3dK!WbhxXSo)vI>95G6zvuNB@Wc5B3IeS9 z!<{?GH3nPqc0G%qQjAT3ThCEit@+L6scd(S{A|n3I`?Et%&QFRc4gnopz4=9bc5{t zolq;%&67g<>+B4wL+QosuQYS@c}n783vxayBjtK?_L&YfJ*w@}3|;)H;>wW8{i}3N zV*`(O7!qP9MMC4M7_P4Nn|WplLCoLM*V&CD2qLQ#ubBPWSV@V(!=CB%_v@G9KEPZ$ z$PJIVW*!tyfF-FJaN~8jKAU&cIs_?5kCB4;KNPNPh$ogH7gsDiIMB$8NxVvL(Y?H> zg=rr_n=+>Ru>KTJvutEkog3VzmrU8+gzM#?IB-&HLNLx)a`@-x3ev0vB1k^@r&1%m z6j)0&W{K8NJ%=U#mMt=q{^lE;d_cAZ@v2OTmE)5BWK+5GInBp}UUqHue^F1P*XY12 zKljOhRtf)OKH3(pU}to_-iH&BVQ8#qpm5p84TQ0!UHJ z5o-N#-hm%AT>~@zoRzt_evxRO;DvuctW5kXC<2}mG`L)(QwaDckCdF72 zB=V+VzR8loh2_|E6LmUUU^hj?tpmUKFl)8?%5Crf0~Z&BP_QJFfz5y&9?jF=7VYPu z&Fg%0*X{l>lscVD`?DU?@-i)kwCUH9q%{uI5cfF-YerPMuuj{%%gypP#@gMUV}vTq zXxr`78~5LQ+$CFS^Q+dTlXsKH*Lvc}OS9lqj8E{1%-CfrI!E6K^|Z&vVwhn`a5hk? z_I^*|VOIIgn9;b)6PU3`ai3ipC+QR^7=JcA@`yROX71~|+uXev?Kj^LyoUmWy=m=# zWQ?#XrbeF-MI##vz`Ik7@PGYGlGJp&83xC0Yo1GlGzOm2cRN)om)~lt3nlk@ayXpt zlDA&WmL*AtM0j03-zN?+U-v2uY7T$Pv_CK*dX@gf+`xwiq_(lm$6_hz)Ba@h(2~C) z`@|Ylt4b=UjD3{Mk)4Jjb5|`;)Ox_;>u4a9kaphh^-Vy;Eyld*3~BpT`Eq6*d;-;| zsJa^fznp=Njq-B1^{7g)y&7B7k65$XnlW4+Sn$VnYUg3||Iu~%7HNaO5oSSbrEzn%cKkAc0|k)A1Rjl7uj z@Gd4@E!CEW+uMNV{d`8Esd6Rh`!Sh<>gJI*l@R;;4_n0}XGCMBT!8Vf>NgWP=FU8a zr-=`FmF^vq^}s zH>-zW!evM7#$G(h21iEmi$>wt6XxJRJE(L4=S7^xP_ALmAj>*EOTeEm=R#v%HN}y3 z>1|+-PaCN;XYQ&F1#fzz!y$}QHfQ5_`4hL_6P6IGtk8viIG&y2>CmNMH9ESyi@|Ag z>HV5KV_*Ir(96<4X8wG47!U0XmrLttg7Wb&s=haN^LQYlZ96#5lU&Z}c(S%Wjk$F~ zwBNf0%yIPr>Z_E979i9Cufqjp(Ip0q@6qGwQOp4?TMfQ~A_oxn*s~8gZ z)Hg;N_+Izlt28I&dk#;cS^DW&@=MyTrBW(A#dakVVgoDz-tIOl$D$(?4r)jjg zt<9m{xLCDSs&8B~IQxh({an<5y1z7GKTNFODUhC#-q0V4HD|EX#dA-14tZW5`HH@* zG&h}kh?jO};aK2O(1UAGT!&otWeH+FERiT6jI2tsl5Z@|@tviJ%ntfUwq@USNPkX6 z*v?RRqQ?pdwhLC8C1$`GS4jvZE|$RE@VwmrC|S1aCz9o}w6L?yFl#hYXF}#Tqfl|= z=&bh`mq!i%O7|oV z%N8nO+hyprWtM#h_kS!`foxGIZ8?{tJ>B$_s#u9?=4@nl*oU`KT9+pV-@nQjJGo>F z3F^Oe9Ten9m#>(~^_#f~FMhRUNGR8CK3(g&x7u7=76@>^Q=sMyhf{(f za@la#?tnG*w8sCo@(zOV zLiV;w?8HVomv^BdJ0@JB=V4n8;1yaTTgway8M39~o-|zSBC=Wd_mjluGR01x@`My$ zJmfRLmeZ$K6;l&^CT4KuoSw_sk*Gkd=+@7SsB`DBpiE|TsF}B0?gNVG&?0u9;`$+> z>}1gS*heA#f0b7|lBW`vR3WNB-X99UbX`^o$al`F*-)@v^PUuggQuQqK=*~A97{Z7OCPMqdQp2!ddb(*3Ou* z%{J}dS+1THURv@>%U7?Dx)^W6rT?5?$oXcw6jDSEi|*e2jKh#A!O!)W9Eg)H@yGu) z^%-kq=x68H@Q9#S>Y7>vYtQW;o$BxOJ5Aap)MXS!x_Oaupeg@m!C!`;)lL zUkO~W(JbuP6NmD};n!4g#=87xZ{3EYbmD}Ry}r+h2bFLmLQj+b<(Y_OF^spGGAP_X z;@!2lT$x2jmOw&dYUmGbVG^`r0+$`*R!qHbJD7d1*ywPwnJY@Oy1M=+0Lo%82Q)?& zd$4%NNjn{bmjp~`1j0*XT9VGK;@YCgIiDDjfEGGnp&Bz2ViV#?Fso-`hlMn0$)wKI z;>!ofUVgQ3#d}S3nH0|OKlc(gB7(6?a=;S!gg6%8e$aDBRc~R75Ki>VU*U@QS}2M> zom2m>b)k#HKicsv!@C1HoTmWj z2-B8I9rq^fru%0e{a9X|ig+}l!3T6Fth95RPE{Z5_WY15W2nMh^ihZm2h8T>eAUOn z)kJP$sk>J^Z0UcpJzu>reegm5dl(+_rFIMteaqy}b0Qw(7P_(btaI;rIu*pXC+VHM zJ+qNU6V;{ZFcuA6I=HCbEwPaoqZN9&c_Fsd_!0t+Tr7n!2A?VV`mrsqy4WbhBmd{Qtc~{g`T8Wh^iCqn9RgdRgb( z-QB&rugYa&QC3vY%;9w;f+W-UK+E)2kW$HNfHy_P3?-3jOT?tq*lv>=5l^Jy1M1XP z5ex^pihxrX>I73L6&+uwQd7L<<)->wmcC(h=#EmKffb~PJI+n&Cl#d)36x@Q8n{~cah@ML}9UOTIM}Q>QeGVc~2^K|2=Eb zy&u?}_i(YvJYPsZ7IWBavPl;FE1-;lxq^Kv7N#LWgdd!sdE>k$wI=1U z;p@o5D?(-Xi#Jzjr_O=E@D6C|7?xy4Jd}Hy`m8*PITQ3)FHLUh!lZbl7}ezwEKulI zWny>lr4rrmpHZcai@={v(Asd(*X#Gc9~)tq6r1=jA6@hx@0iy{c={%qJA9>oI8}R9 zyT!(}fvGpZiGAw(is&moHTfd~91K*yJQZ;zQstck^zcWT$g4vBzzMP%}I?;f24rip=^e-zXC=d)PJG@U&pf11=d;P(U z^dSeDf1^qdiX^Hur~l^-ey=OX???h@^}BFOz{yC>=73^H1de;#jXZ@&|&$E8ihW6*9!(%|McjcB7u8a>wzsBOwmXeqc8e3HMRD&f(NCa z39pK*_kOq(k7V=uX#ydd8)~9oc$?eGNOT^cUH-n2f?}d=#m8xs^Op}MFJ~`zgnW?> zz)DRO-_O)*x8>%)csb-Um?rG- z`G?z@@kcXWSE9vON$j|{vwf1>WQWX4G40$3g&9OrJZm>n&BJ%?Y$Up9m>8%5nor5v z)>3-^_vJgcqdHBEj?${wC*LMEjUI+XTUoRm!_`c?kV56ccx+ zm3_{EM&=(@o%VO3$wR4Sadtq1l#TfGu?CwakI18FQeo(SKL~t+3pz(sX+KFv;VCJ! z6Tr@6sZ<;ELhx}Raj+i|ZNwCkwDUa)Kk>h!<#nfkvZ2z6OTV4h1^%rFE6&=Im2)O+ z@!7M#q!_verT# z9{5L_fP6$>PWS7xZ4M&i(S%q-mdjW7celTQS3SM+6TELoiQlBvS9|}E+w5i2Zq>kG zg=xp&Or`m+I2M)9Pc}VXa@T<<;$Uu!#J85$6guHzfE9Ug49JZ?eHWl^k9V-J^!YbZ zw?6>F-ayJgT0%g*{-!~zyQw6iclG6-Hg_DeT?S2>4^1e4j|7r;OTsfKl6%-4XA47X zi>vstuHpslea(Qc$!n_yjjL9dGeOMko94j3`H^!Q?UHKn9{U$30KLBpnX-S=9*DDX zu^MCJ?$tb<^2cjf{5%DKEW;Ih#y>P!dRKwD%_kd-&kO%WD7*%pjuiq}aIq$HB8-so z9?Ydto|33GD85VdZT|M z4*G%K{He1mP>7+&ijWnVeoVruD!|=*k)xl8?YaOU_Lnw)+d<pH4@dd^$-Wx#sv@KZ#big3!m9hTRbgHdmoFmq6Ecx}jn%JS- z_ba_p`*$41TW_}CnL$u`-JEtWhl5eK+)q>>dZs~SgY7|hUw(L12UQ0syM#J++2|$P z#^S86`{FjYW}%oW!W7js#Cu+;WRXTDymYI7-B6y5Xg+V+PkpxNIgUg(HULfgEC8GE zQx!QMX|L@*U|T+ct^~pFfLttiIS52{ODqD~Ez&2?N`Nj#FmdeJJnc$2B~=oMq~U#v zGvYX7n$BhifCA`a#|KLXpZh%?FL$IS4lq58?{8k4KGVF&o=}#ulqy<&%TCJ6 zxr;8%PAHY-Yx3iC=3nEn>f4vDpzMNnE`)&28bo%ujsf-Y^BzCe6m^AGwh4@vhD%_I z;mfLjMh`W^lq#$2rv4>YJNld^wJ)A^)9WMfpx>W0pB)quueBNc0x(nWXpeJ5~(~9Zd`%$dde0Cf&PIa*$(fYluR!V zzeLf7lDF|CVMsAx8vET>L4q;-PbxIXQKU$N8S)JJdkd;OaaCX)9!y6dHKYF-#HKsb zXF;#*XVOPECcQ2<7l)sCAx1|yxzD5c0N+^Sknrssip~T|c;*|+FZZpvdlF$W=mbKo z56XFWeSirf1k)@%QmPUIubXjrW`jg)1wl#KXBwu0)h?#;GvzIA#VcyBA>PUoKiZIL z$hRmh(XF05!z{!qatdJhTdvgN)*D}7qW$<*{6fj4PR}Z5GKnBrAE|{W$nj@9<^)0PtGQ#}HH>{CpBh5oGva5I_ktj@*2f z#%de18y=mza@pXDcUQM>ubN2utZu>ESr> z7tLpt<22+#2_;Y{yuM1i|3D25w|JNJ;FYN2&#$ZM+u@5*4j2P^SZoIfT!}=KC8(ZX zC3fE9qH-ZmDY*7?vO{xxvOdk$90C+t*d8d^c?y)&$T{QW;UW#>%LwPEsj$2ONBW1i zqw6FO)p^h%dzsOM%z}Vn3bGJoV0Yd=f;WBKv>~CFFPX?jxZzdxHIxsvQVLR0UZvTW z^G(j}MY_o3ul>myD{!9An(H~cUcC2aiO5;1&i%=F@|4+7wZvScow3hY_{B9VI*!#* zjBG)rxVPvK5of>z`x2TT?Euh-(0rMgd3kA7Lck1#Rl0xHe8M=MI$W#kHNZnpPJ8oi z8IK^D1R5b%Gh>F=ktW9J{cOS2e4Mpr)|%tlE{5WpI?5Ky?f*m5S4Ks(zHblRB`FP} zfOL0*fYK#MH;8m29Rq?=(%m7Tba!`4HpOsr@Q=X}Ypu{so0oUB39x$SehA%DtM3w!ijh6gYWlsbAAslDc zL#gZlgGgToPb98=$W&j&VmUc0%J7F|c@)=iUrx66m}SOi7+<<@68V`G4q^D^x0{S5 zf$KfQRs73D?tvL03D|9>qLnkt zaVMic!D?_=s{Wm2a^F7p_e_rP?|>*;4XhR5M2yrjz|> z>=)s!ks}Pjb7`jKacZBP+I18>PMfOmC-RRLh^Il-XRyh#$D?DZj)>`W$a3Taw#>TJ zUj^`D?U|#h&m|Eg2oT|V0w$x+a2Y=F1%^KdZyE3@GQz%TOf?NG{W^J?{T_SjQQ{*4 zQ_`~L_Y2q%`xafSC}6}}^%6nG5P6Bx#2 zKcqs&eJ<@S-tU-q<_wN*&Xcsxvd*8T{|ZeKynNr;=$J)jK{3tX;v`QuXIAij$zV(W z_X*yC2$I8H5s~~~7JwRxfuKHx_hTqxU^Nd!oC@&=3tc1u6s++HUyi}s1pOj>mnMYH zpL!5ZMuPdI&~)-u{xi<>)5REs|9$2iA?e?g5=D2krMhVJ`x*3$jZ_-cgo>XySaDEj zahW+X{{@aXeniR&gwpwE;lTgfyh4RIvIXVt*6!Z!WV$~hEH*|R{(2KhzGrt2lK4zv zO`^4wUuH$Hr9C}{<=TzoWZ{Fy-#kNnBtU?;e)a9sm6D^yYUPl&&D)5!w)wRhSpprR z_6!gcefhn9G^84Jqc@}>T>FVYg`|}Syf#?`9Htccl70W()$F}HN-~XfWR>-hvbD9H zavb98h}t;kPUS;CVN766^rfG9xKL?K?_o-LA(w%OUNvCYT*yrx zCSK0%t$42$AdcSZPu)_Zl4opeD4!)RolOwZY8Xi(utt6Zw3p8JXoXjAsl0Y+&cS(> z!ZV}Yg-Ip4bW8RC-amFgEmLX_Y9%{?qwqeWmtneME? zY_0D7MrQd+G38JD9}shhH(d8#;3n;mFGsj?OT+Uh;*0_GaQq{5-JMu}ta#&n!bLKu zi)0P?`gZmQ%k-1UIc@7NTWo1(9GjOk%uKllX2 zd~WjQE6F;%u8uxnQ0-C$q0o8z1ez;gUK~sF+Als5Ixy(0HA_a_Lb-v-E68!pqBI`s zP{PmT65?`&V_!y@Ep2jBIL$4!(kh71W6BIGygGg6d^PVuYY+hrOKxpTWa;!{`NzRADg6UH~mIlf86Z-VIw4 zx)9nuiN(|8-IFluMdLj36W#U+zPIJ*s)QX-S@s}fj{_*o{fB7L&%rvt3GxkF_7>}Y z!ThJgpj>mcx+o~V6Z6;r1i6l;pcGk*u7T-sn<%yv1j8M*?A`R#fV*JE_5o;Pn^F;Q zgva%~PkSVWBpY3z{KQ*$fq#?Ro^Qn@^U1fy{+QZ3$(`a?CMQMxK{4%6{#cT(z-eE? zVN1ch$(73bgQL%U6u-+K!b(J7Bw>a0cv-UY_XFdt* z#!JnxbDl-Z@r4j`Na-_V)H~_Be@G0&Oo;68NAZ1tJ^AgIH_y>E4jhU#Rd0zR#zsV= zyD}wj5|6k-n%M?(E1VPTJtNQL`T{yf3Dq(#%~vD}o!8m}lvZ=DCmMEj3ktIAC+PPI zGIuHFyZxE=KGX>8zP?~Ko+;I_iwhe4gi6bz@I^H>h>YwkY9$zjG#@>t_UdkgU$z`G zR=H>4VO9X$mqAiToPi-^+@B7Sjf8W>X^3UYm$auDI5_fK681xf3{L{oY=#yv$`z0j zt;MECmEL&Ly<1bI9Eg?GEtW`(;D1wi&(U|5mvuk*d}@2*@wTh$@w#Dqr=yK1>fL{L zMb~UW9r{V}+C7F0KLDI6+_LHN805GKRk9iJPqRMq`=4o2tM?0DwU@kU=6DfOnKbFGAvZ+9|&e7Kx_uKgYLG6Q@wfe_PZgaS8)PM+oeFEr6}UK zjkSKSqK@=hsF33_2H`LjVRAWq9l6g}oGO@ZK}?ztLJ<8FOA2-10`HjN*6-IsJ z9$^R_7S=sFt(n(G-McP0$O^&V#puHm$fGu*PISjK;s zLp0Cq2whAlLWfD${9WW!Z>)jDX|7Gnwhzb!(m^Dk<=N;Akm6e-W0FK)pYKtc$)vX} z$~S&?65s=nTb(41XJutFg(d`=*7yAC&%pO(6}$+O6HAgw&N<3IH1iz7PkcQ^6}iQIq}o#9m9isaE8c3d2$;2oGu{iV%F`&zCXW_2)sgS0#mnqJ zIpxuyt%|e4!UfXBB>ksAdcj7T^mpF&4XVcz6qFPP_z~AtS!$u^uVj3qSXCLo<%Df@ zO0Jc`^s~MbM1P`;8m@uiW8FKn#`W9Z9H6gS4XDR^@n%tJZ<+VUj!0eg7+tvCVy!9tAcR<5wo$iaNo2w*0y!H|tmXN3Y&n`8=6 z_dh6ls~!Pta~t?~dAKY`UCiHNdBkX^&f4U_4&qP24EjHlEO)i(iqGFQoH1Uu`LR$Y zLJ=IT>bbpby-Lt|D&UC)uyvlRVxs4TLVKUa8m`_z+JGB$=>SYtqYd1PZ7slZ6^Z4~ zUdj#dk(x!nB%eTfFjxJokUas)I15M^W(~kmT91U+B1A$x`+(H&u+$EKEN)o7HnZNX ztKcqlqUuLa(?|?FtnPvv z*9Qs3F=+Wa{^(wNZE^R+D(E@WrcZ50#nmK`JZKzYRq_Kbr$#6b?l|-}rpY5w1Oscs zpJ%X2`o;!5?I|VETQf^dd^Qr%uAE+zuVC3C2U9pG`YFrdbeTO;0kt(k0ldwKw?YY*ZrxPjd8Z z=zR8424jiN0)eks-0k^@C_R(W$72q8^jw|5^BXR=1MRL+)L_-zH|E|5i~VqGi0}gU6cL;8a7!g2-M?RejnuWYg={!TIloGnwIO!DxAr{!@EavR4%M0$>8gExtlv%aS)(iyGVGmk$w> zN6$&3g?r$jifBKvebxCaNCp$*YnZ1GC@k3Hq#kJNT@u185%akN@F>N}K9~rce{UF0 z3F$WV{?kU%8F|+@mszbchzRjyoBe2$n35tBaVQznV_~g-E+7=*yXe>2GK8?gT3p73 zVMCh4AnHyufm(C&FooP>m7!)#SrmECGK^Bs1*yM8hCh!Rn2kXQbbx3!ln$e;86 z*?lcTt&Lo4{`ybyXd;dLh*wx~d?ooR+bkc%5wirZ9NmKU)$aB2>gveTCgt{k$Z0n| zw82x`%3};5S6h^Rz$rQc#NM+Pub}~#nZ}090w5V82o@TXZUY1cxOT?W+s|v=zI`rN zG6`wkD@Yv%v63qnzCX7lL1eIX>#f(1TZ?UOvVzokE-@Rt)<>rw#bwFnt&az$TsFOZ z79DA9pX|+(t7i3DXJ=XtbttE=Z=EfYo~y5NsEeh1vhcYqzP%f=R-&wGO|g688$|JB zF1zhj$UIBHaJEq9x^(?^A6VD3s`pD4_AoUJvSr?XhampGiXWUz$1z+u=W>iR5=!vp zL1_2x7PJ^9aR#l%$pBwmU`d;^%=AffDVviYn)7iZ)MVa;q%-#zMpX*jG7QD;% zs+Z1^4GhoDf4_=SUQIR=9P=E+oa?vyTfg*C3sYYa@bn{Tx1Y|f(uZC|BLAuA|0?K6 z%#<%v&^38G!^z*H@Y%9>uVss}z&WwEA^4@GZwihy>|$6;PjAGD8%AJU1-ENA+T@E8 zFJ`DY^VeOwmxpA#Y>GGJ=kFgC)OC3?K?%?xNKh)B^Z`VNpA;ZNQ{rr7zZ-@JmKhwZ zv@}&EaL|QNakV33FbTS|8i)m$yz$kkhhs}(#}OmZ`n^%$l{R8cCED93La*U9Do3hhi zGjDii_N&K}7CZ|q9*HgPzQzx~L5O76Z?ve*sZA(gh)P)UTpTpLw3{k*KK@-Q>@`=O zvo}{m$B2#~y~(MSV-|yVt1ALJ#mcu7w(zYR_D{G!u#YW zR@#l-@ExQmh{SAZ_JVHO0*T7;2P+@H%O2(+71T!9?(Qe@QOq6JIDgyXIkQ3x#L`Cm z%gq0)Dsr?fUe0o8lc1!5P(V|63n(8Ajew%}B z)bNrBm9=8DQ|JSR#y>S7p79UJe+5&a_Mxi@q;Cex5h>GUWqK=q&MFkx78m2{F=)R`U9sGKU!Lj2hWH5O?i2kTz=HOlVpY*GF8kIR-}g3sM*9_xkYSY+z1!v9 zv>s*Yvp19wXDDqMCPqF()HXRLOVoBT` z=(P=P%BmxXQ;>x$xxk|$h9QElMj2AjGQOjpoInIT}{FPY%o z7kd#!%rykRI~+77BRw#IzI5JsUgd2}R0JMs6L0$sp90^(%yeOg)v02PO7T}(Sc`=D z*hz~to_u=8{tfyP!yC{3Oyslc<87&Wd6s|M?2mboE~ci14CuV8FNi|JSeRw^aDg$g zuI8)&q{Du-)LW}B3!UR9g#^b*ewWm>f~F^(eDp7S?MM76*C{Kgt+{s zeym5X!b>pWSZ3EWMY&X$g#O)L?0MX&vFq)hymGeCAW{?P$wnxy=)59jQ0OUPx{xz`Q_Uy$KxIb1f7>t zj2NT0L<+|f9@=o^+OD~WBhzeRm7E4v>G;2lcG7hCIOAFxdqdF7ItJ2_(6cbf_628B zkV>XgYIvVpv1qNQ5x<9gc2+VOV{zj+UP#v^ni=LzCtmEKb#?n$faE_s>w?^2zqL*T z)z-nCbuZ%KS*~R#yCyRcWV1IXDYhbK2oVN({3QS3ogkUy*QoveJGPcT-qDqXTrOZ1 z6Y4uT5UvLFu+VCb%#cb7vH9d*fC}C45)}KWmY#F4vcb^e7j29O#YnyrIWowyqco`I z;>XOII(d&cF|{`%odV;gL9GRS$UDT3IXBQ82c!6!87>5DHt*Hl9J*Jn-EnUzf!Vd> z79^vXgQ5hC+4<`LoKa8{ALUmsP7i}vxL}b_6ZtVex^9m{qnV&KrsCnaM4SSV1H?@V=t2+!wGn?~cS0gxguqz8mw(#O>o;67(Ho4U z2dN1!@*kBGk33ib67b)L3uP3fy;p&wHD5ow38eAGgZ5+>kBhsqf2C z9RP1WB2N550+_0bJ;BDd{BWSgiIARwj_5i{h4tE;bh2f!4j@v0L~CO9GQRA*9fCn; zL#qXKg+v~Yxdhk(LPp7xwo>){!o5&3%GXPG07T5^=KDe&;eOxR8 zNUI6@>~p-*GUW?vxsHHa13L0P>t8#c)f$C)dO4qmtFZ*;1=Xt1Xf!UIyXR7MJbRlN zh{ZTlAx`ZmN8N7_#ej-M94%^SeA1?dYX1mI5v=p+_#F4ZmR-^SU~R>*43UV~T8603 zxw!;k0EVoQ6NCS>hza07Skl8bkOK_fOfW=!EgjgG)IGEM`_lF{+y`vS+FY0A;A=Fa>%J9Dsy7!)0~P=78E_1a^A< z@%?zQI{|G7@&jgah3lhbU2oyRT%>nSf)feCPF^}GsT}C!9%}*C?@0<9T1M0-H@k}) zDIi2nR@&mx3Plte&}pT8zpq2E_xMn|CE<~aqg({_pzOJ(cZe!-Lfjwxu23BJnV8t- z=0U@^Qtv;71G!SJ?SLsr1+nK%Y45~yE8U&_l?ltmU!trlHjFxx+mgF_T8x#wvLAdL zg*CpgylMvf=8}g0L8hKt;f0{3%Tj`?)BIGvfQf;yEe;)ZQEdgc;a7Xx2!wBwR)2qb zh}dJ_AnAo?hv{Ngqy#Y_a6%YgJGW3 zgnx=-KODKYIUD&+=E)1>Q!cYia+rsRhXRVBw7A%sl=Ny*I)C1t%aye^ANESf-}=RL z;w_0J%41=r9beNeFMsb97wAvTm&eO$O%+ZLdus(c8NrHVg)+ZFGX`$x(_LZ(z@itl zpOA(5BQOdU^)xzo-$nJvzO!^g;koBt!v*t*jUF@2AXK}@sn!HF&x*0mxNm%R+5|tb z;j|>HsRHt2oTze4ST#TD}i>61l|MO-Q?3I?k1vy8Y!h77S1*rqhgN%jz zKHrZrJbrQtw4cJ9q4N1$}}6jx}BypS^)PncOS2(6FjO?dIZL*b}|WYaAPLTO*N5V#b9N|{!)qM-$3OK{ISUsB^5W+I-X0H1@7s$} ze_+xCWI@rUb%QC?NX?BD=fx)4{QR6jfJn{VGAJpUT+4W&n6sBt;5U>PWqR=qp&{;CD&Wm-D?Mii-iwi=NjP+&N~VFQ?{*%&>(}H zUkrJ29&=)L=jMbCLEA+HT*Wt5p8?IE=9Iy194PR3vzcm;ImW|W4a;t2B7X7atAleZ zV9wOyb;C6Sg;Rp9ae#md^ZcX>2c>3%NdzLXPv5$u&G-O_jZ_$WDc`o+s<-hrrVoi& z&*-GZ9i)OJ!&ra@gJNF&czHWm(}MA1mU8h5eK&>)(%x;fxWjs92PStJQF4fk1Vt|sYw z-fxz~C~21k&b7jh8`7}M^TyvP5O3&TFI__@za`LcJ*Cgq01ZZ07CYc!wEE6qte+3r zyO({x-^#~_djI?f%!bb9NVl&7V2GtNxP4M#;d_cq65v$N%6Z9Bd#Ww%h}ooI5o`vtW}#BsAiQn8>qk)Wad98}Dv0-}&l zXBlF8FjEfrpeEx&pbLC|U`^Bp6gU=uC{fgcrBL&ftb@Y}6fYCO%^oWXMl?$Tag*v4lxe&H>h$0)uxT<6|II z(hUusQE2zojen4%9EB%E?pOu#vughFWs)`7pQcvVkYL|b5^90d@2gHg<~5{3fOv<9 zta+n}SB|RY*+3bng6)q5rBfvmg#e^1uS{b#PonScdd@VeK^c=ma2l&ZKABVc3Z(qh zJ7ZqLkAgnp7ga#=$2S6}XV^T&LoV#h8c%KW0`Skf8PVo#}fGh44q|H@CI{56B1{u zj98}k`wgz@7cOeldp0I;;^{kn{UT6#d~w_^1Z$$(!`#uHdhiw?>UegpyEe*p0FeN) zM+vE>XgG+ZhL%`dl1PU^l%_F|a@pRAp3TFKbMP+M7_|k>tct!1(S%a1fcbWhkVrC> zBH{TMS0$OpdIcv9E1UB)RL=ob<7#zQA?u>z);#L5TaMtlYqVVi=P+?sIi0IHptL0Z ze$q8J9*kqO8+|=Rd6rE#6og0MPyPA4auX$s*r7;CU-Y&t!7{IDLY~;r+fm&!fB2as z{1y@h5qHh!6J6V^h2D=b5-+!3z<|jSlwfbQkj$h8bB0=#?(t{nh%yqG>@OWK8L{JA zC0mCc(?!E1C1O`weeP30Ph8`Ci8Xi42Y%O;Z4e!Fu#E@**!pnxj`)9?Fib5WJYh4E z7=h4RgP_$Lf73^r5}FFPEbfwk*Q%8L86hMP33#T)m@yC(n4}$Jv~EFz^v&J|sFm7Z zua>-L>mrv&{rgPIV{dNeO!shhoMLkJhP>M(Jjij+nP?`M_7+z|7}=LO62LB+DzlJlJ$n_e(HaD?p7pVS!THAS8$j<{ZVs66mJI+g@IW zB9uhgug`#DCgrUS*Qfc(0_Gq~Q-s2_dsniS%)P;G57!ZZ8nO5iH0 zr_ku7`pd!G*xO~T1twL>(iKY?kw;K5;**u?~c-r|A|p;N?Eua|AswFMD#M%kQRqO?<-2uKp$~6)~*L+x&`37 z86Dqpwms%28aP=aHXH4Ap7nH_=vfwRSok&`G7)fPqA<{TwBs^UlTfZjKNs`%|DlH_ zGr3U3Z-%U$e-&1N%Q+p&W#E^H>_as~&C0m&!B*C{V9rw~)xOmeV?6DcCqJ9qQTJhp z2g|kvP{yv2wLNV!nRcEvW!FHSRrFrldT%bf>a9EwkEPn5M^5C*3b7&V-hGI z{ULbnZ_munm7X#*D25o|ZLto&7Id?(lGvlcnidzQtsk2r93O!3R!C&*gxgcrLYoz? z;DdOq{Ypek>(xUf&~^k}s6r59O8p7E;Ae|UX}*MiS!Itylj!Ose~$npF&5TUaQq52 z&9{74dsBT+si!378P=Nit+SlIrRc2Eo^i@y!#cJR;6a&YV!n<^a1_5wP0BXfbKKC> z(&@n}gr!hJDWYIPGjp7TbySrfSEKg$Q{SrNXar92DZv(ek)*mW1t+cv-=l%;^$F0T zA=F+){AMzqR;($%G$ivFeYs#elmm8oporR~bNlS9C`3B5$_Vit`FY}T+tXtuSI~tK zQ0Qd2MbFtEKWtXgm#6!cZ{-!bjYg}P8Tu=tEP20#h9QDqN5ZH`kz9AAqsV0;4zFVLzw0r!M zoXHt0hG3p}o;E?Kg6{vUCoq$T7j{-I+OA!gDV(H>mrZ2edO?LnDg`5;+j3 z<$&z8@vr0plgn4Bs}H>bw03`!J$#;{y@%aVIa0+Nl8Uma2CK z(OEA;lw^J4ebwYxH$>cZZDO;Y#Rdblz&FO0(7t{hdB*}7WhDgH+Bh5a5aW>iknE7E z54TUH$ZgE0IsFO16b0~W#7GTC8- zvW&k-6eRhj>yoCavVEv73bA?qKpY9UW`z{)fq((7QT|+=D7{sCd&fB0N;*my#clQ$ z%L_y2HsC7wNpHy2rU7(qw%R9a8{J=adE?pYpd7X98o5l|M0={^{7G6L!FUL7~_2{hv zHO?^LLO(*nAtK}oQ(domDG*#hsEQD~2S{uabm(&)zd79;0Zi&q4#yUO#Ig65f*yRg>N-LI^lOww}0TPiDTQ|CSwgVycPoV0N=r{T$RXU z;(Id89!_rf3gEDvpe1i* zAX&IKW6{QPCLT!!dB5bmk?=WfZ@oVs%vIPiz0i_IxbVI^_Ivt!cDsD+Teo+y))BPK zq5Hl9iL%9{){IsdSf-&YJ8Edx7?7@PsZhZo+8NO}$5)c0VEjr)h!o}n99|Xh0hl+o z{1}CCerf+Hc0`Nikpg#ZF6tyJ{Hb%P{5xQQjfG|#QYtgzE}7yq6|1kdFD5;lRRwTa z0%goC7unQbW=V?s_GDF-@34x0-NfzI;%YE~^>tlPnrbAxKj?ds5E7|*8iDD{7+Q(>5rp5MRA0{higiF{4>LyZAW$ zWK`@FJi#_0Jjwd^v$>Q-WN7wjeUzq}i0RY6En(JRzr9JF{`(wZQWm<~gdK(`7jlLJ zJ(?{Y!)*!|Y!u)G^4{m5BKsGvNkkDrIicq$GNpyN_T50H?uSS=c6MKKPN(}@d6*pO zFZZT7O+46h1?p7cy`o)k``O?`pptmV(Ge9oW zV#|$!xjz<3v?#_!M#keDmAIJnd%!v?Q?bk{{}bKy;x;Nj2I9n#XY;^o7X%K?9NDwY z!gLmU|A|F-cqWG?XW>Ju+47MC@{q1S1iN&H8`s!WyM8CD>?BO~r~?F#ie^8MIQ#L-0gQ1* z44Y`OD*44Jpm~KnXPr+fWcbmPOD3>hWAercf(4$o^xS_c4tzbCS>$00Vvu`HIK;=W zVmeZ{t*4z#KT2xzA<==QX8~e8mB_43Oq81FC?ujLM~qS@4CbZ65OIHMsiom>e}H5b z;V>P+120Qy+&E`5c;y|fJk>pg%Rc+lbC@x8-qE^#`O6m>2YCOFJ8Lid5E)1b$;kpA zr6Y`z=Z}NLHIYdzkcM}MQKmb|~i_Z;eW83Df<;hF8e-dE92#)MOwZ zDB6Cuf_4kfwn>!0X>)k-Xea4L$0)*Z7sG^;h3J8m}1nW91FH493)o-s?|fjZwVZub3nz%-Jc z!k%92KJ?djB*?_90u#E99YfcN`a@9RP2uvm;J78XQ3H|+r$o~+dviE9e3oM>__B-XxWs4J0dEy_n5 z?z6VJb@K#5>9}Q-p%(Zhl7n-usB!j@DCoqgy{yF--k>} zvhsrLaSAuAm+E`bzT4!N6yhXK+DR83^po%;`3eXbjy)kw%n#TqTj%w z1xS#DoY63_nVTA*iAnqVKHaYh`2}-#@<;W4M2@+JCHx|1>RfzmP{OT17^rvLBu@g_ ze^pBkOta;?dcezRr;X?rV43C!)yf{x+T|MpP=y!&{Th;>1<=6nN#Jki`ktDfz6cYu`MUh#<-& zvgC`38c;tr%!4f|-PyXfy;rE zgp_c_5e(X?pzdXr=jf!W#zs`xzF~`~4rv)&hvUrAJCfV%XgEp;8fldVl7h|)kuMTv6Hme`S zAU!Cj>(&9CWD_>EE9!X2Ry02W+QX^O{tt2Lt)M5D1IcH5*``4TaLo4|_UCdhv^v3n zGkVHxma=$Xmn*|Aj)uFoIRzgO&QloJOJ>3l=&6?-Mgs+glDg+4c_z82QrBTs>u7J~ zu^5CEHP5_Iq}H?%;B2{($HDd3sf}*>6Z>wstZwBa$&r!oGR$IPBJwWi5r!MCC&84A zBV)$enYTgpR-V8^x>P0=itTzW{O@AQZcc29r=84Y^BZ4E$Tikvy!KOc~W^5&KA*lz;E2=a5E7s zsaLE~`L0+N@@0ofznLrVN~@{{b0U0l&bjR$;j4^{zZL5aZZy1km*%J35QT*N1#Q}X z5Z@e`f-ttcbdopAW}>0kUxh+-gGLHUBG}}ThXyl=d49q94t2ZNe2wu|R)P?!y$*gO z>_IAVH0$h+XmgboW0N?QSZgpYD>Wd5z4=BO&r$s8MPHg+fVf>ZXV;0aKdwy*0WFdP39Usd}AJI)(G2{ z++0enMD>x$6#L-| zM*9YK%PVem!}s1gW=y)1-;Ol1j-EjRrd?ebz%W~JoZF=^uPMGW4$X?-*BW-82>n|0 za8=jF;>Njg@3bafjk?`lYO?YJS%)y_trSu=%F7`_k=L&^t>CFSnBhqXZV(7!4dPk-ev*LC+a5q*qt{wRkb}+n$etWPvz+>)=-Ns)_zxBuD`?n+Z zbjsGy5ZH|WSE<0p9|GY=KLgGd=lky~4qaFyE8h<@Q;^*FZHh0IEgjxZuP@f$PF-HQ zTF8&+osSHUDIUi5SX^pqF+Xp&KR)w6k074+F>0&rYOVFwev=xnwQEK+{C7fuf$3Lu zvre@`WU=n2l1Bee;&96>Tr^3qcLr=M@VA0pgGMo}0~K*7(|l&ts!BD6yG0eLY)=+cq5iJAZ}4;cA4a zDuLhS#h>jDu(BYgGdy%~1AI(4HLIyS9)E&J~+zA4!^?QD-d(%KkUI-7n@ z=;a1G2e20^SRQRcNmbtb(q^65;dlotvm7 zPL;>ejusUN6C~LE&Pv!n^`ubo#=@n|g)@tt8#k{dN9r3=^wqlr@gDf#vUo-26gWA2#ML_vvAaxNOBi4qgS%rR^_g%>ef2L+*w2gl1G>vVAZ&k;x7OCCs^7^dy z{Yr7k-rRs7{ZaRW`j1u~Pwfar_jt`=O)1zEG=WW$99XJaA?*1thj(&62 zGN#0oTQ0@A-<`v7CA^+42Xh0{tRIX%Hq99idkNWg!ZS`a^tv1v z4K=sC`DfKmt8KYu^jFR&JwBw2s)h%nboB4h59QV^LuwTcK~2Q+bF_c2V^_8){iy_# zO-%$F%f5N-QsjS4o{HqbT}wnqw%!?!h|VljE1+)`62f*VN8*_O-ds5(E3+b#{{~Lw z3mPUdXIhwbTDBbXJ47fN4yX{#@-#zwf$=(tUoi?RvH7EUGKlV1 zy(5s3B`-$4^5=9kQD|$i-xB1DpBOeOJ{JRtqemhOle>(WNkA4-07a=D^IcX*^$Ehs1V5cln@dc51X3VI8*kN)3n$ahswN$q zF5Ahr=J+aAN|xmFBfld)rkR#cjqbVwF8d2z*f_cZF4a!u5pNC@c(45t0%y+N8(MCu z*!X;&8R0wM@in(#**E2-bO_$7l)Sn`c>a}AWWkj-xhjGZsd=ISK^Tu82W>ao-b0B_ zWYG13;SL>x+!i^a^Gsi(;F_4UN*r5%wAtc;B~}*RGjF7QJo~D1jCeJl#Vm$E-E=&D zLnhlge5`H+_%1)C%D!w03K!2x%v-z8gODR`>y6)U@)K}W4>n^Fe4msI*>3al%%0R5 zKF%rCeptu96#`W+>E=-apXCE-z z=b5D;#isO&NdggB>GdN+X)@!2IW_KivIk$t`ozcYCD$3U~AVQsCsW2|q7twphYWXzzBR16Bxhpm# zp-v0Oy_$M_Fh8h_ih>#UQ;NEc`Gkt4ko>5ijd_Jx>vx$TeU;w~BL|*^Lgt?(+&{X6 zn0|%Z)$-E_`wJ2Md71_?9|9VCpT_M><(MdR1v@K|CoLv=Z`YI$Gj1eS0HrIu^A;Ha zu3O|&RTz#IK*QM@Z^rx{t0+fE5MD8sfIB=|;G`XaYlWDN+bOGJ5x)U17~uzQH*UNP zA+kAjNIMUdTl|66#;$<6u12@L>xa=JeX`*AxK6vO z4KY3UoD2$9#Ey=URr(vY4AcEY2jMm<= zlN}0T)S~qcc7Pe1n#!zdj#8*$&Zb6-QcLtwD-sJ+^_2z#$VI>=`qNO^I)3I8v*oFN zQ=p36Z=#wjyiYvBg~hXEi6GU5G=tL~1)*sqXbF#x!lTKw!hvWlj0EqL!w-_mVeU%s zO&I_W8^c+iTLdq{T7lcQ>f*BVsR21$|6{-cxt|w*i}D zyXYbCjzEi{DRvP~40>Q=!%SV_R9tgC@0GwN|6dk>jX4~(p_Py(6Km?!>_b`E6aLL6D`40r zvnx?2JEWiPgY9}cWgr9_hOsop;xyvyQ&z!Gk!n<%?3pf1sE{*5f0e`ZNy*SVpbsxO zf7vf=g_p|$JZ4V{9jD7)dl@q(3**2qoLi&2!4rir@-Ee#hB6@vLF3tLRE~Hky0io` z?=ShXi+&+9qa&vQ( z>l+0lm+Di4T8Kywu?I z!$(mt-}-S{lKz5AMs2}3PSy$}V_JZYmxRDoOcg4-=hA&7o=zzC`JOG#acgRcr{}&N zCKjKhJka95?7|JSEp>LgFk3!42ESZfZdk(Q_x_9Q?P!itJt-4Tl?Ki!n3RQV44|QYIA?|Y6M<0^R zPyGYUZ?5NifOoW;rCiUj`O_bBR0D+KwYOklu!hAb0d)oP|D)-w!=miIuRk+%Hv)nT zAq`SW!_X~V(jC&>44u-YG?Ee$N_R*~OP6#=cfU8!_xE1>&oy_Qv(MgZeb&iNM~mG3 zFefFRQU5VFTl4%&Nhpc-nmQ(u1~4J~!G1FtOr_kZtwMy4+Qmftsfr0R8Y&7)iF(DT zL!PT*o&K@HXZqVzU|8ky2Lb+tyf+K|ZjOf>w%*`C_u}YVWC50zo@EDCB`lYJ_do3~ z1Hf4NHM|L#Q}FdYn}5`}82`xlwO2dGs*tuQY`@MmX!nqPT^wUiW%%e;lS z(7A>c+)1w+B6PWE2O{8t&X3r}-O`juBt-~>R{AcZXR`riy0Hc9_C1-$Y8eI9EgL>Xu#Mk_A~ z2B|uM{1qV6RqPx@m{+u!#a*G)bgz?yy{kc3NE0vSnMk7OWgiaZgm}2Pim!OiY}fx3 z{uLlrpUw8lVzr#!GyWSUUsub{p@e%D2H0Bu9im}qYA?9z*0<~Hk-MLIsos2$X=O6%M}BhYsb_N&AFZK4XLjGjGHPOf9g2>%ntQ! zqaB5OAYm-w{$O5lF%0jPMv;AU%+;9>h^Tn; z>0-HcZlL$YNKIbfFQP@P>6xX$qJ6mm1U(jYBi>KOO)`xxaDJ3L@jsmwDzDnk#oTXe zf1~+pf&Cx>ZA8j;>GKU!b>P)snqPLUwuV2C*z^halt!P;J|TS>mHj80B^_!=_KFyj2^*l8LwRyPHJzi zm0v)R8{-~z8Ce&VehG4vIbNurY9R;QYRE4*Y!?RVGKH&u(iu4dgTf1?X46HYYVX@; zuFT3hZL}2iW*>MMF#*2=%!H95EySQft-)dfVXmG1L_?b(wS^AR{1Yr02nG>_D;WMy zM{vXI2;jq-{2GU@diSmqagg0uVEtAYsdWVe%$eYiHpQU88tiYZG$V+}0jEhh>Ccm7 zc8Tb(R_~N@KQ#l9Ws6-{n&(9>8DVa5m*K|NEYBJ4Rir%jnIGd4M`~${-atA-WF(0s zfuF;qGe5PE-(}8On8wAd&~oAPU0o_DjE^vlOYubaSxsxqo4Z8WuH4Z6xf~=gT3pHY9PxHprOI0OhoJn9x)bjRO!RsJ#;D^gqrf zD)#Con9?1d1ju>j{sAPM--+OzPM%l(2aaT)+RShaM#|KF3yei;&udF`E_VW*Kabig z+FhJVU3O5_bx__1$TUCX27wmeY}JbO;D-Z9o;d6LU}5GbO~uma+wvlisNLTI0U4!_ z{v-?DmOd@7ScA|Kn!FVtDtkKM+bU$6N@xoD>l5GwKvCk0mI7741+c6^mlHY&1C+^Z z(~Ya7QD^{Ai5CSBjtTMjrcRmFAkm!F&(nbSj_ssTKIe@*8v{nfHqW|d+;-e$rny|( z&PBGNaO_p~$dGn%cp6JkPr(X`|3Jk7R^Kza)aTpSgkMjZ}w z`F#`#tKt(0dQ=Nd;5=qd0a4|~$A|ni_Vzu$H`_%ZoOnsVFpPwYZ0>vSxd97x1Yrm! zZR==!k@x@%JF`p;!rm^>P&oDAVPodqWdm5q+5x(ZhKB*9t%7x=_=xd?9*F8a{V}Ef z7exJ>_o-yWMOmuQrHzEA!Ll{iqloSQ$f!wF;YI4o)=)-d6CDI3NaQYKKCKu*H&h!D z42J~vFm$=X;?J4lqQtkU{6(B|Re3u!P0AeGMa$PKnlz0<-U+bfnf@pnXQ7i$qY^JcYUuInlB26o2;3#{^XJ8{Obb z6rpEKPW%R7OxT3ue{y__Uv@E0fH{Ok!VYzsrk?oq9&c**QU*X2S2;d) zAoNkywTdaF3OA1ix<`Cgke-jnNJ?&`ovEKYk>L6_0*?l;AtWmXikdO`#l5G^-AC(B z=L?smg){j)WJV!qJ z%#SV2f%d_`ES`|5NRUB9c~5|m2T{{F(L=wuyGx8-=tUI$&0uiTYp6_?QA8y`xj)@9+!kaG)KOufD0z?elt52mVL0j{=yhsV@14G0 zfOne%`;6E^?Q)v?4qj8(g5Ql0TCj-}jKSRYYy;sBmYA5(`gRUXdznUvuMITFnFyjF zg;Dsc+NoIHMgYSi;CDwL0>=FZ<2bcaokq#;xaPSq@n{qIiu1vj+l|u3fPmALjleMj zUKoE5Mn|)mQei5z`hmU<^W~>S0eD%!r$oRjmqt8D=4J`_zByWbuHJjd_hxpl734|b zD?V07JP)XNu8ri;jZ4j+v@Si(Wt!(@q2bO_+rhC--^&<+liLCB#>$DfHx32PiJ2h=_pEkK!H2_VKJn1TOv)_ zPk^56(cV2c`hFG>*U zEU}UnAIgr^>emXT9O2)-p&U~~j=JA8lsCf%?riN1Swj%?_pm5va;yMK?p6ECO9!L~ z+CV@o6h}+@mBl)KS!(-Pj=cr!c22k0twPQlGAQh0n>*ZxdA`)rv0lVpr#bsOUM6nW zeAyhPVHln^C4|(5J8m++z=Mpw&?h}5G7`ip98%5#(&)F;Gylwp9sn|RXIDW!LkazB z&<-=&yMNDVwZYpV#^A7UB+b%m5buP7xmKtaO&COCJddzJX@*bfu_Rs{R(S)9zoXXR zK`|3Lb{Qa61+tugC~s0+vo<2)cECJ0(7t2~q#8Zj1z4>P^=e2{IBGN_NNR@%a^=*%5oxbBHzTwoLt*-0Ay6kr zv+H@zOM6`AOV20Lk65o5PVOAhc=GF|{`|u^=bO`Kw&=e`{ZinK6MHR+1Uld%l-ZvaIh`cD_Ja0k(*QE3n#>F*Vx+ou-mbiiTN;uKJ@w+WKFnvj3v z#%5HN&)%-!3MA?7R!I(z(U^zBP=Z)V>VXBG_)4Q2tp({ii|TYZfH;O6kX1zP5EiX- z5m(67SQHM4>uwk9kIg)HR1o0W<>Qth7^DyJ@UOp6uzv_B93)bS0b=^q$W1|6sR5~W z4B=a;DPc5}YaZk<=Pi|(2PFUPpPzf-(f*GQ*5d89*ZAc_V z(isT7w|rFnrV)S6ETZmvikM52TC=XR zndVy<1j}9*O86Fx&`*#PQ0^%N2o*EHh~CS|Us>yByHX-;T@<8IUq965JPaRfemQVS zai?tdP<{czo}ytnnaTp_2Ex@kq7L8@Kxr_%8oeObOKS$on?e0pUM-chETx1o>?CNv z$U8I8`GV4U!+(G~YAI<}> z({@{_Mr$)h0r8Fh!S8%hvhNIs$)XrAZ6(t&?gl- zf1D(ZV0FP{C9AHO5nomvq(YF@k9)y{_k^GxAzZI@zT{VcYCaj6gGzPN2~-B!Brvq1 z9#PEX9j|aUFCkc^H+gR;%4{;s(jumTuET!h_3n%-a87&Ls}9>2qUpvOa9yN&i8 z$4+=(M8{6}-hUOr7w=8MRT4Hxmzh9I-z{Z9Sox}|Vf&di_x!{%=DGCcy~!!$=(WYv z@gW_k${{93P7%Wg!-^evwGb@3wog{ob8vBui5Ti!X@=JkV#;kG_r)Z(TwK1+`0;}MXBmS*+scP;;{Q(&6Q~JO2wP4r)WR6qgd5||&s%iSu+Mhns zBROep%#p7#tFQik8DB1BFgQO~l~EH3!M8dnU5HSeKrUCe-d?e-s$hd*bpwrBgy}w$ z9)$YG^pY9%YG0%en0JG`#z98AcYkpLW|L9L7zh786O}SIZ=+&}$^p0@SPIk1mnQTt zs7diw(GC8z!@tW*;gap*9cJoE{xlxA<6Ul9ldID1ioHF-i@UAANzL!`bUV{N{G6TE zn7}xJbWMXL_hy%d@?PusOycm?L6CpTuVHU#b`eCG3 zm^SfeE!m6_bN47Yhte})8{#N1Kv6EjDj6XWfxdt~Xek~hqVliz`%2KlDvxZsEIN&@JA za+kJ~=ho=tUrdaz``}B&Fe5N@$5RIxjHYYfXfA4PyKg*CK&~SKj5F+wUmQerl%De$ zDHFraTVi+P9Y1w=WfP=t3PIx6WD9q8{)1CQQ;Gag^Vfat?|c9RQ6%Upmb>(4)M4(k zEzIfz6+5ycYY;uidfo$1kbf)PbQlO(-q?bqMM>kKMy;d8GQz${(<6a#B}8*c2n(P) zvbKPrDzYCBI8%i%KnNj35^}K64@44ZTOtzwDEGbVs+UI}R2v@oD40hsTMzyU(+N)I z!P5~o2u>yPq~%_r%+jQUZ9PXe8?3buJ^)T{SLO&EBe+2cvRtt=Z}Hh9$IYfl2mCsQ zYlQvTsK-F}^TIJ|RuDg}?ztTl()fpaWaiD@vd@iv&JS!tba*On>}U$oea? z`o#yZEYfdp*(52xvp#k0IBPKK-W#$P?WcGl(NJRC=Emqr6qrk!I;L%e8Y&9T!@Kl} zf&@bX8PJ&u$WD~!#Jqg3|AxPhw9ZD|O{5mBE`eeEW2~@ynDMjgRp*vj84oZS!Modb zU7YqQ+vOK;VPHs=HMfbU6ox7l6vqV$w5B=?% ztoLo>`}flIE5N!xGC$%-?Va5S2Z;j2zZei2X8sXhc=n7*lGu$*bJrAgc>s$PI8UOkA0HNeq{Kfp%xs2Z z4_|ri77&l35fYrm#0c|C;na>{zpCzt2o>wy(n1W%lY39RlF|kJb*})*aZke6@~vr4aPrnTQ_sY2 zJ}!zLiAN|ux|LvW1$%!JEU=p3k@Gs{impIb0gtLY&XW~Atq2_Hl75m=)vo{U=l^XD*-!*awYHbFu#2*)-^3KXm^w#_v|CjFp| zh=dOc670_WsV53fmC~$(r~?+WJs276(-rs>aSQSG+z_VxRp8MpwlRmKs`!cg%m6Y& z6L#7<3#Zi-_e7#(pb-rAjeo;gOam{>IY8QN zIAg(3@jl;C`B5HjsG|I%Tzw`Q5%NcEL!=vp0n@5M|0iO6WU>GXDCii^KRvX`X`;OJ$Pw`Us%^!iu=q!Qo!vBx3=Z+E`F&JfpBz~&^sIURbFk#oO6 z-|K6aznp>&85cn$^FwMketvELbD@f7A-Qr0A%Nh_bTDu(xHYsl!dNeitd^?Cm;F0d zw)l<+92KMpWg|jGKi?I+y6J=XX3{H0@zEV)A^E96s}kKROpTaDek{Q~S%S4sKsSI1 zv&JQV^pK{10VU8or33o@vYMCFWYG@ZTF?c#FcLXswd zur--b7m#p1+)qp=FMTy&f{5@FSjhYS%(@uuR7^Z@O-7%kN9fVdzir*=!U?2BC<5Rp zmh9Q)Zn-dek+%4_L{bhK?^_GS3yalPG>Q$NxD*mbJ2>(1N4#pX>j;`y$7$#;H5{R` z^SQNTNzrA4i0ipqnDzlFNE>%<90}Ba&z&tQ4WdKZLEJhw%) z%rDJTZ+Em%OS8kDH(@Jh_QzKgmpZ{TrT-}f-y`_2rMh3{+hxO^RH)2PDW z+0Ot8OcH3`UIJt~oiUvX1%a_?pAUCom6A|~_p}VDT~K5Y1eFoQfTi$TmTDq6QAYYk zkLJ5_lYwfJ1EDjvB<7nxTw#6c3?~M1W$_M#>vPXXbvxw%Aa3_FAXlv1P5H1Jy$}XS zwRd?PA1-E!1+w~G8k9;c5?{@*SDoW{I|D7DkKpUEz-z)`2^Kb^1k_aVO#JoO5J^-P zR8t20P)bkLv%IHv`u`Kyh-gzwjFM$>S{xmO?)PJ zfZFki5vSh_@G?6_6IAcz*Zx(ilX%&yCngAH-$eL)W&nql#dH1aoNLRut5H&9bNF;c zJPbz1UzJFOfkg?W>i~z!yH5M)^l*54yY23?_B9m$>|HdDuZYX@wO@;vdKT2{gSr>% zR1pkLlG34)S*$owew3}3&jioL#gmoagBIn!k8H2>q&XyTcU+iWNn_K-A?#@lLa;6VLP6_*(cX1IaIG!mulw@vW14BH*zvPrfg!^C{T&m?=qYx#^a`%b z8!(*ONH!B4-w!sM4TIJc928-hO1|rfuewHa90^^exKCBsAdB))ON|o6LnRK5e(i~Z zG=LI(POI{&5k$+FS(n)7QS*ADm)4>j>zwylPaXYwNX~6gF0mJBVM>& zx_&An2E`Knxe=$LmI;K~hUx*yB8$ciAj%L0F$=44`umqz&V!8h*G_;;^RdvP4^yC|v(UIk`Z2cVQYSLC?c9j3utp5xM{a~2m8N4w$rv~fO_yv!M z#+l9ySdd}nqHlNG+2O;5EmAmWP+5o(xN#BpK)4W}C~zPoqNgcmw*C?A{zkmF00W-L zPo9Z0##pQ{c)_}XtN!9Yq&{J8EqRD3z&`8K%`VZ?I~Ncz`4?KHle~ zpp0jVBVXQ|zAN+_WH!}VtrV3YIl5hZlU)+mSQvDg8>U@%4q(?SCAT>yYeD=)iK{=- zOW$c=~Nl{SzIYbQco^a??njCB-A8CXZLLw9_5_&8%Iyl%*ooMny ztKdz2r%iT*Ix#lX=z{Ly)oUmXWfP!M&eT{9!eDcZ`|1dGWRZgJNOEGLRc|~v1l(l+ zak4IYN?|U9VuYbeEoZ=V$Sq0p;f#FS1Qq9%{v)T#jjh7^Uw^qJk-&a+6Qu_F+)XRFXP2hSn9w1w>xCcC(X0T_s!Kf_Hy zA1Zr%|LR2$ecSQOR9SaM{+P_-z2n?L4#R=>c|8)0T|m6anNk0pP^xCif0>_bJ;t~) zI8L?Oif1snNRqy)($TC3%Z2Eo>yfh;FrInj%^ZJukMoTj3+x+`b4+Z&3{;D0 z(p?2%{omLI{b@48S6RSm3U@Hq!4rL1tz#n<*M?3U|h%MjP=q$^&7B zAlSVW=bLyBDrU{dZGoxG+uv&)15@?Co+Q`du&A&XzWW+`(je8gXuQ-4HQYP^fbdRH zZ}SPjT<&DX54vJ85RL~I)CWSq_jt#$n*>OSUd-qGYCv!a19_O*kmwpcddB)R2x_nd z4DuYeN5GP7CPD9pu2m`_flb8g&p4VcK3HoxZDJdp)OTDnEHeYBcx@vlTJexT(-&h= z_t=3{kf(O>>{`sO$bA}8yeTXiu#9LEi^7qg(8eL@to3PNQP|3}VfIxEkcXAoEq!e$e-h=>j?OVuZzFLp0(Bi4YYSJP3*jl z(Xe}zOz>LF**1a5q!(Gv0Cb6&(%$YqTQCYQIyazlF8D-g1J=b0&tNzgFT}-qCBUT) zkxj!b`^t!mIy(NM5&NiUh{q_=>(B7Z;gl-m%ew9f*Hyl0W%eVy)w6ZfN9Rq)^@n4u zN6u~hQ8Hd|@gWD3w2rdQz}7)B)%-yu0ezFCdcSSMk_j=55@OJtV^!2)-YN)Pml?*J zhM1mfTeYRKddz`D0YqQfw~N))8>^te$5*0L2g(J5H*wY8<(9t%=JUjzuE|F~0pG@y zT~^XzR6<(q@1IU=Mir3_tmGe|z-{cy6>Va7klZMGeM~aD?9< z-#dRPOhW?gs^5xtaRc)kWHX{*5Fseo7Y$bw#V}PEjv)$VhjjSzEnH{9p}$B0lwsy7 z>@PsQJtW5GtSR7}oKIsAzYghHN4ZFSg2OJef7M_xupqltMe11KGyZzHE`YuUT7%6J z60)eU2o&>lWa5T-f1V5gY5QcN)j$w7bi>NLy>Sk(j&kaH<4f5dv7C4YS$``?`O@`?&%@r#J85WXn(Nj#cacd2j&Z6>n) zxQ%b^AG77pm3D~hr5q-b)w)(z|AP(bZ3d8Zzx6!-hO&B_?-_Sq0o&J~UtpHy(lfuL z>^?bAU|Tjaq6rug6<<49MGm*Wo~!Y0kXCo1f6+lvXu^uXsbI|!eyN$N6@&c@d*=gu zL78|E$3906`^A#DylZSUvPgs(fK8T)Nrs#Qiw5@8_fY@(;1 zAkb-tN$a4hu$ZVhco*Pyt@>b6Nqw`$R%VRmFf6WV#8q2)#iu-sTp)BhI$Fyg15xMR zf&`I0nx;51MmAb{26k*})%Bp8uqZ&FWE0b8gK3X_ydKzaqN*Q3^6A7dBXMxKPx*D$ z+mESdp9iGSTMt?ki)^dErN`o^H9Rx9>nDk~tJX{m4+4f@?)}t7qJ@ z$>h+^r_Z^ezk^3qivO-w`R}h$n9c(=0RuayZrY9 z7@x9H*{ZrT5}uP7D{DufEGr5!7%$p+l?govA*dgk%MeA49#jH{XbtL+iw;${U~kH3 zXz6^2HQIi29DtTM?813wmOY$h`TeXnGRx;^k<_O*is(tZ+61X=+!+yG#p@KQq6x)4 zj@GC`3LJ8Byx#BlQhL>MjOmH6S58+~@r|NUff8eYxCV;aumb3zhwRl@qL2w;xkwC2 z2D!6VP%CtG?YJ)revprATjssCetxg}G!zG@H(^78bS|*%QBLEF()Yz^1>!$7 zvayr#jHcCt>F<-ZXV+FL&StE?BV#p(=5j`fqr?{s4_r8OtkPmKl%0$y-h8^wTIYN9 zlH|D0@{j3j-yWJ*D)&zZMQ6ZdslA-Dt|5uCxo)5WG@X!PlN}}*;9S?sS4u>uk=)>! z$(nPPxlO267G{0NEg%|Fp8B&8Z>&TWG=%t`I7EZcLH8RtjqLipz=3+Eu-_8^%sOfT z7??+yj$LT4g$Y&!h$XO?H}06eh`@<_fU%>_Tn^97Ld)|Nr6QF522W}d%Z(c<<^hFW zqXd!6ze%0cm#7N)N8l1TEESgZDEN!G70{LpLc+8;OdsTh<+IsN9T0+v-8-Nx4MhQe zkrtXhwEpJ*m>Ud|%>CXL_ZpWW6B+e90|N>1ZT~Z3fyJ`{7@i(Ipzq+1^mw)Tk=(#h z1atpAY&V%Nne?GuTCM0k5|89r3hnD4k=xfVxg+&h3u*!GyRDAlOhH!O>s1tySw3J} z;Rgx8dQV$h5B85RS9A>c^8q`kirmT_D<>(auCT2&~`77J$o*3Dg`2S^~zpCi^A1kIHf%2r>OCEa`Vl@gm&zLw*<)ovUU(Obq zl&KvH*%gJU2&^a{uw%Nxx%&u|*Dy4Y$jDXS&3l*;OW^qzLit=yMb5efUh{K`4D?JO z-kc{o+gNTEg6CHxjJt}lBfkvkaN-kQzFLLIQkEGGL*fCt7f;L;Mt4BjxLWttm}Hg` zKcGD435Ky$@kixI9qlM|zrOFcDFc9I(XW+hdLi0T79uRAJ)%8I99zqa2)mJ1F+Ib(c?5*KlNSz;?+Yiws-;YDW*#AGhxM4g1_(AqF(C ziDXYj7>$S=^SnaRdNrQ1{PU&#ASVe&KwhxWVgnKlo!W#iJ$7XI@j%1L>>w*#EVFTk zGuZvwO%;u?$#BdrO06~(xWEsaK~Hg&14qzi$dLer3=7f0`_rTJ)~e(=F&AUO$%h)O zso9_F0O|itkuO|%xC5v{ez0K6{_F5{$Rt-jSVHbsKmQRCv_?toRd2AyZ2bd#T;rV$ zB>I_i?+c^bN;kk}gg6!jv8r(jsU$rRX0EkW_wXL?{ zG{yHUV`Z`4H@Iyj+u`|}@+Kd1zhMM(D^j(oxel;D{!a3wZ~BQG7>j_2WutiX+c~~{ zO5j6oNqL81qCW^VMuQ1j^%I?~cc8bmZVaYO=ts<~+gN~Ct6)vo5&H{ctmU8i8|G>8 zKlgyvqFuqy!keHUZ{3m{V;U$>HOc{xAGw=i93*>^dZF*+ZxVdW00Od8A>a&sl_T<) z$`2tHk2Y1PMo4-fJb1TYJLv`F6YFcr>5`wmc{+VlKbuF>PW2+cYYi_dF^BdMA({0j zxfLI~54P3l{4oPK($aA6<=1!RQVKw^Rr&(7_X1w*rSEF$zD1C8E#Y*<;X?#vhzrF< zH6v0u^!xBZm)z<{+^VyNd0zO~o34&3F+5+aTY%ERT;}APGn%mvvbHt}$W^QWwUQX7pKdf!E6_yioLdqQc_I0D#c$m9e`=}8g zsD_NngTU|sdacA>Km40jx{kL@(KIw%-EBWheIDEw%Y3(PVarpa?8*E|-E4Y<&i3g? zI9JTwg4QWVcs1rECSb0&wqV4fKyUoA~3SkCmsYjSh@w9 zZu?!Moc=T;#XVEL7jN^ur`N#}%7D*Hyv%)88=|9f|7l$mP7~V#E#NEvev04xi4>Gn z{&1R#dA^3!dd~5DNh{qX582~Vc)7n1SDg;esaI=BozSkGy-9eqdAe}1d5<3-#6cWU) zv@HtlKA;At{!>36#4qmVpT3rBzj{{k#ho<}1z75PG+oOo0g8O1^6(-z0{FtWm9+Nz zP4lL-_IM(TqG~^sYREh&V-p^b$7og^&mabh>z$0+@u%HuJ<51rq!&8H!D(0ezuh`% zwc~QZVa4mRHb)=%OOmI-^U}zF1BZd)FZygz*E<9gOGuE61*2;@{%p#=K=G z{%suCE!cKulNK41{??SiSdgPx%0TM%D44--1)_Fg`c7>rMr;qr%s$-B7}&20PG)Ti zZ5Y248hlfCh20A-|5ou(7wTu z+l&1>A(J_mawo($(C?38EdD_{m$6k3?)kS|=4!91v!D4iA10Kf=5E*_pIb*O0s`K~ zzD>t^`7XK1NGy%}6>cY z+}zb!6ZOg^Luc1PY3A#mWFo|yvCOu8<>Tn{5>nMf?zt_nGMCUW(IfYx z={3pGjDjB-R4K{?Eg2#3{(zHW7{dJ>s_eD$aOjc7D7wkrVD1`^=?oUJSJv}y*fr~5 zb`V{)ng}-RcwzJ#3UN(HQVGlr3~iF#p>q3W=s{SPTLb#kR`&hVPmft1o50no%nzM@ zOvyL~@@1oe^Nj+Z%x6_5_R{JedY@MT_-y`~|0$8d;EHEm;lHQ-3)s&milLl8kqBze z(=ezq1o9d_n0(wa@yk9wfcTp?87~YM6%iLMj6q^&N{2|&%J7q$ALTe{@h`_Z>Sxjv zZM^-?P}4)T`IRcQW`DKTCPSU6pQGKX7>MDU>OO|+yhcoY>OAcjpFY)0_FqfKPRNj5 zt=R?YV#f@CQ~h*hDh>KBva$!UKk2&s^)dshBi zlz}t?k*70}H+fZU7Znaz7pkq$vD^%Ucx?!n4$4k(p^T%ynZeeu8l-9fWd|wSLoq=U zZon@Fb$Iy~YgqjK^4Lg>IwE_~eqiUo%pB7@TQ-oLph*g(d*)s2R>$_<>F$$>Hb)z1PJI|yX9#1r*$+nIlBFap|=uIK|UmdcTi6AC6my&Ny zs~U{-<};f1xBIuK!(F)iA+O^JL^{lfwgeIAs`Yk%YCo9Z{ma5HIxEnInA+7eYyi23 zdQpl8KasX!7>HVpK{^E>0jFCVlWItW?VV;4Z-7{%f*78Pck)6@0;eJ*#tZq_@=H(2 zi#Uq2NUyQrS5scA2$b3;4D&i+DeN>Ba_#>WefA@P+rn3k=9_E3N8ha{N#erCRHrZ? zI-o#UaCZ?L7Q76UHLho_j`x3d>b&KEI4S}J$OZkbnzpjx&7hY1tzCQ-;nVy_;BHS# zJErVA1Bwy`fvjV-+DYmD%2yH2`7xTo!k5)?ZXz2&LU}1(YUt|JO}-a! z1J1i&M$~d{r<_S0^Lay{pc-u6Rd~LUp5ESPdHtcIDAOVZh*KO{erJS1xj6a=1P?>< zUvA=`5UCDL_mjet_nIy*B{4R38~%4WYw)`E5^hw)2|Ij>E_{-*Il)s=5o6_{hhD;} zWNTw@R->^C1q>g`BhB_SqeXzr;kEtVqiIsG%|f?GV-@xhFB3m0kWP&NK(C(s_U}~3 zs8S8?$XKe9qvXB^@Gd_=xj=;dN!Hg`{%>bdS}pnax8cI)c_XHjce4#8=$5Uw-*?GfhB-DTR7dspK4gC!B!<;qES5mWxbhdddd2ZBA%f_*Pl`kc z9m&@F^yy?)XFof@cJr|B9-1b2D-VG7jd{U)`iNL4K`<5AYu?5`rhDM-`1O<&_U$x+ zI*R{ zidgWzk1Fn^n{d62in(P77Bv&#({1m?XN1xnUv<4`tNJHBY$?&9BBA1c=#sN=&j2o7 zx&FnII$Biw^mz2tc9#F5wZvA1XXms0LpI7B{%*WD_U?Zt-nu_qlrR6nyJ}%Q%M(Q* zI^4I6t*)T*uxrVa`YC6laXusTgt+hexMJtYezL$}bHQ0o*}_iBGhws;oDsL{Pj5_@ zV4`S_(IoOF@(=5~fxAFhr|)i}7u`$3v4Pv9Ys85wa1aP8ZzR$V^AiU?CmU>qCF+u- z6qbQeCw({_#e!x9ol&sKL%+1JGscR*B?e| zsALk`Pc=uMBX?I`p;-{!1M)M&BI7jgE!7!9LQ1=@X*+xJ#Qq(U=&{s4f-&Gt>M%Vw z%f0KBrw2<(HaAS5fYnyVZUKPOZ=%O-4b3&zd{*uZwgJaIlie~r<><5ok>vmo98m~c zoXOpn9XO3V(5xzDlLVfinW0H2f&%j?pL5GH+=Ton%Z%M_?lYh>+H~Dfbg-u z)BfRuK6Ve3Bf8CMjkuHh9Zkik|7C3!j_)c$?|gTu#=2=W-rI#iw!1_8Y2y$Vx3Z0` z_M(4MBSoqTh7eMj^+|^9P((Rw?E_1+vLRW~FYnNFboN`XCrNd@x0U^JTtKE)YCI98 zu^rQ{@(wY4!u z-R3NT<;MV5|EK+Z{`Tt`#jKV4I}d^Ohfzuo|Er9bT5Oi;@Ai)BujV`7OTvrm(YT}OaXTI?}5exCd_T&s0Y<8>vWzv?TP4@Ksm zn~J*mZ&kskjle8!@Z`Vs^g3!3&;{G{G1UQ`>aXQ8BR@A!_XDo>N3I@EXn;GrK+hj= zYWVoWrHu7Sy4YoZ!QDS#f5Ej~ZusWjwIVd@CBK8f%}v&VtCF3QI8WH>k^iz?gvjpw zMzeme_b7@(9cC<3_!3GzHww8-Urq~~YP-wD1^JbG`c0^~sZTyU4^CQ#O`mo&_1PPGDPvY< zp6br-p1hhzF4`Z=J@&3&WWDr1JIV^UOQUpad&~-83no2fN%(UtZ2s}=W)1Ew7JRBV zbm4qP`S@p>`s%{1&+TD5d!u#6r(HlP2g|e3ValAc$k#PAuvI5xM+V^#C*Zmpq21p5 z4?Gcj+k+)?*OG&jbF9dtNU_xLKs%h-vV8Su_z4U4Fe&mdnOLNsDl3>1ozlh>Ee=Os zeKXk^3r3C<)-rT<6IKXlQYSreR-e<^1A-eWW!G?0VZk&F;VeKGREg!Gyoora+p;b9 zUb7erSql*n4HdV)0PPwa$p4%~@I9tR8~jN-Mitfckf)-b9358};100$r=7C<$&4RU z>B)J1?mK8mD11a*mLZI)l+6^FBr3|*$I+oRHnY;$Z9%)XNkxgfQ*?7!Y^Xjf;%O~C zt4V7ukNFZBZd_UxY4okwfb3l1iq);+ijOAX&?HFLjRIJ%qjNm!7T4kgbQj%+Mu_#J zSAlM(UA;8@mOk|QZohdsFpt68;s3J$mWaA|?u*XuR@^7Cp60TsALB;8KP7$a3Sg`$ zuZ_T0w5<8!#(C=VJ?~9{@LtQ+tYwz}h^f-ZrYB=+={|oWPtMn*Q=RPn0hda>^rQgf zmh*!WGw9R&Ok17lHd-wY9F@6Ob(;EC#5bz2!a(mA%4dy_Uy_?}K*=9x60ZyN(yp-} zYJe(CL$3@DGe>^pd`%A3)`ciyD4`>+jS3FB671P2lYYeQw-e8RTQ<0k>)&sX8dd^O z)}Tm`q~<@&U&ulpueX7F--^4R@ay8|cfX4AX>K;>BW!FPV1`=)&wuZK4R?fckP{Y8OTUa!?|Vy`OWwli>|HJV)aPM1w{b^~`H zZmD;PWlEI{$Ysd6=P4WMuQ+q5*75aUR~t;?@atT>_ZUdyA93Rw-1aKLvM0AN+|l)J zOvCIrc^|GukYmO5c7M8p*|;hs%_OS?<Vw8VC1NealA{8~zAle*QYiF5r24DB<^0{YnT^1+;`9dMX`{rRW5(vgPI zx#fiK6%piSnhV@jUx-P_U!rF;EPFuH5U2&+I zl2goQwcmjR9|k4tJ3RF1-x?PSBffB5v^GJ`&ggi#dU3_1y`H2jOz zzWYo6UjM4?kg_q^mCW;#4QcsigRHN$iOyu+P(tQN=^3cq^%fp2TA(cHiiSa4k8n|X z=$Z-kEobJ}xJ6_rPuiviyEETVfBv~C*({r(c*u8AJzAznY~$(C z)-D!2Wo98kpOkxi@iw|gLC2|iU@Ezn=1lIVZG0pgR!O%vXQ?)GTR;~dFYL>8(uNEt zZScfFCYX8M#N;Po=9GR#d*>cILBF6M;#<${g&jZO4~@6yn1vwz$@A8VdzRc;e|IGl z(GO!gs6I@h(-^D{gqq93uj8!Xao$s3qq=?^!y$#9w?yfxv>l|a5!E_WI^m z^{k%x#U_=*Av~rVQ0F&8Yx%gsd9KvftIi?x$q3m~=wlpm2Er^SZ?0ZzIlu&AJwdR@ zt$F-7#tAM9xeA!~F%LDP#IIBy&+M~YK-xcp6Hctz@ zf2Tc^Ri1tx){j_AW1Q^=Y{WMjJh7_|{&P1X&GzV?72Ub&j1w(No=3H3tcu|X*DtfT zwcr1u**=doZ9G8LDQ-w2iT|(&yNU>8v;oVfh1w)`?pC=;AGV)-a?oRH061mKmloxK$Zii z!hV?KQik1I&xKjHnZY1=j?0@T%$;IM9FO=TYS-J<=bCOk`{0@P z7_aB-&{m9%O56E0#@XS>C7E?pCfl_u%`;g8Rj~A(+ARgD^uz;(aC0K3ywYlZcllT6}(JQYzViz5br)5vM}@O5s*QjIY7p}Fw(D_;m}lVbrXp7J z%X`w#FS9UG4G7@ZI8Vnog?1oGvBGqzqPt;<#k=JzwwmmB%=CAs{R&w#quH0$b7Z#w z1sN-&B506WJ(VQrLC@)$>0~LE=+je7ic z!U8+DVwvq|^jCwBoVS9Tz<~+%h06<&|3}kVhDFu3U3g~b8W8Cm>H#E08iWDqMil85 zq@+Q5014?1>6Y%U0coU5P`W|7``vuU@%@|MvuEyo-Pc;{T=W6tmsYBg9II7i(G}jV z-*tA$@5{Bn4#3AJl&P{hjL6uwUQyd%aE1wxk6ax=+;*!IP5XAI+MFY{setl5)h(bQ z?Zrv-P5Vw#ulRh8=p-E{iXQK9G)i1T2{~|x@&uye`CNFnlTMzrfgE5D5U~-2g;HG&ddEhs zSk7FvxY456MwL-=(H!9NuSL^9*DN4&tkPr~kY@yW`Kp=EJKnPF3II zy^*q+*F{cA)DH!%6<}z_+uB%kjIV{JhkXH-9ZU3LnKshxj#MdULMKw2Xye2tp=+YA zPXUd@-O0>f`xu>D^sx0bv@ZI+<3h7bdf)2<2c`oWG}X`auhi#v@+_F%IHOuKd*_r< z)^f=@ms`HcA{`gswQSPLAA0SUKg%GJp|O{nAm%o+B=FwO0eaNejLs;xJJ^JfSf<8| zpRMM{<+5q%8{}BWhY>2!!Vgn8X{PCuw`#8=sK!)ez$J z`L1Nye?ox(wZ;Z5;Y=Nk^Wl+m{|uIx!pAI_d|xP&nvpOfGYdm`e^;kA`}i*m{ZdB5aZQ;_=RdfK zxW8~)C|db^^YQY#XY=|(cu>>p?W0!1FWH=4<&=hB)Z70)P<}^=NnKxOc4T9IgYIb( zP63`|LoQJ+9t9DmI&af{UyVR-)4Y|B=c~L)Y-r8@s=vC~g>7N4V6dI%JTo67<+{t; zlk?e!S6j0{y`Nn|+4-FP+pnzcDg*B&v&;sd@V-@R6LNJ{bR(--4?^Oq)nWC;6!uR! zx&(;lNNPo^1;CdWN;FG-jsj@eZ7VOe=NV@?%60V6yH`F+D2W;v`^TRGMs}9wM?lM5 zYV~bvmlid+Jn}s)*CWzunW->by48qyR4P38hvU3K*M{to&Zw+pZp-$vB;@9P0B-^HeB+J)xRJ@JDZ3|GGu68(Q3xk|1vMxA~OJ1e>_9O}BR zleW#1s}0x6TQ}luwp)*llaHP?IqJyuIm}<4>#rxb*ZEXGUu<0bghRhZ4KVl)3`9dw zmYD-IYHUJc0gzVypS@;LtcP@a(1a-P_*b2pdXz-q}fTGXLP!z!XM z;>qGW9On%G5=i=1@m>gF*T!0{aKL!*pG>PTcGzp@F5x|6k;aFP)*B0#aKcQsL(}Hp zM&}z`zV6zMlI5+Q!Vc~nBIn1uNj_JGQ=NNPfr$#bWDkFGms09U7l_*eGAB>m*Mcg6 zn$rk_-_JUXKk~CtAxZZ#p)@Sckq6nAD3G0dT=xDU7F*Pc=+2=Z-RYp@HTgre5f#Y-Q^$H`o+hNW8;foA)$1V3NL8bT@qr} zgH77AZE%T~Bl`0`a!<6cZAtZhQ2pdC@$ekQ@7NHRthO!weUdWWIb6F(Tth!=3PWrP zZ|t69Aq{2ay<>H3UM{SoesrR9`%KH49r;N(#_7;xCDn5Ed;vPj``xnYtDl@MqX778 zWZY&720v=0j!3i7_m2@7@-)}kP9Wy$pM}L2?hJ1(BrpsN?p8^Yeh}p*j{9dXO25e! zGi|!XBl`yfU%8Xo!sIbB%__OjP*+8SL3(6O^WpaQh3)nJ9Q7-}+)lBqp*D1PT&? zvc}tLAV<7n6;ung)%E@FRQ)j?QOcJhTuFkV%sU{*UnYrbH|;km8D%zuHjKa(sURk% z$qk~Xg<*PsrcK`=uk6`inJ564r7;L}1HlTvr1NFcYy6mhq(>mX7uIV7*~dUTy0kzL zzQgL zx@SY*$B8%h*q)2^d#zNz3iW`PXdSSdwVKXeYR6-T*){T1n(RG~65jpcBR+XarT?iC zE?|yLhWYEP^3>S}o+SoX=-_^V`402rI}4J4Glt1X=cbfsgDz&5kq>Iyj5l7aCRJ}| zzPD;{m7$89iCJrA27~=>c8w*B7T@Tv7ev*?IecOd%ksB0 z0{LC(DGe83Vyd$LmrI0>;!^G}N%`7z3XrNh%>7^wR?8H!b`02t$Kuph!rJacn{V-k z>i#CR9)zUfu;M7B#BOFNxB^`5Wjq`cSZA2oJ#rNOs97-~nC z1E0ig)&E4Xv(D-ykNTnQ`-uuY6^w#DObE;go;ee#1OPdpHDta`s4Uh~tu5}W|Pfiwm&76c3mjz+m zqaS9?vSx_O$wh0hV}9Gih>Txva~4@eyGhUe(l zX%nFrh{@h=#HC+nw=kCi5%G?|CI6uH_I!C$s<-+g)EEczg5enfyOE@8h5L@8p^j@^ zHFf4mH9U4dACbjsLt=6hql60bV1{!{b`LU3&&J_Z!=y1T+14Dl;vIktJm-DGWnpkz zn3`c}{OZSxSjU`3J8fqOoZ-AiQ+`Q6nSiXl_;d;^6nAAs(fAuZ(IK$3QZp*_XD`J6 zt2(plmy}_&4-3E3ZwSOB=!nd<=$PQbQ#S8qm-%?{Ob8X2lx%cp9*Vq(Be_Q8ucGX( zp2L$wA=Q6xH``fsFGXaU5_zVYOg~CSp0)tf0HRY1bpUHwmU}=N`P}{oiW&L1A8&*+ zeDNFcpJlZE9u${=^_3_+9UP*$n7BY4PUpo^w%xyHLEgN~&p^%Jyof!gZz(%3#!D|s$5?6_gN9(jv zNV_rG@ys(xIE@?!Pf?ZKY*0qv$emKkC;0fQ80rftYJ4*nf_uivIizDZlAA6w&?fofBj12KTRdz@r&k? zr4WiX(jh-hz5x`~LX!Vv#u7bya}4n9k=9bn(W6%~tb$3WCpzxaeH&rk57NuSs!cz>Sh6NZtLJNUuoB;2=jg0_jisdQ3~T=3yrWeU{1Z9{e5Nwv{Jju!uyUf3Qi zfr2ze``55wa8Ld5hrIsuMp4B%O_-at@b`?PGz3x$Y>I`8IVCeC{+j+e-PpI}vcXe) zjaZR$mUDR)+tT6Q%A{!~=*`u~gkl07WMuUdOk{(6(31Ikbh}`*!XyVn)COFu~2H6wNl=&T*}_?L1T;HS*1SuSLTik;Ia-v{^)sV4EVvyA$I@ zuFL-_GS>t<-qX%*Ir^40PIG{#sg=R5CKiE238S(8+HW&-t^JZ@GZJ3Wt(iAn zvn=*o3rj=dqBg?r0*5cqP|elgP=1@=YD5LvC>l8&3*BH8OISDW%OQ#^0^hw_#p-y8 zK)aDt++)61l)TUl>PgpivyeLgK#8e(gGb}+v3AvZO^rgR$;rkY;{+e!l^b)=JwrpC zfsZ*LvwY%4;;p=kMkYy>s=e;+@{Q#m^-!!Fo64>4b*!a62P;T0i{_ANz?_-aDXVw?UgtjNTP8C^s8x{@?@t ziYmk)0xcxLduk}pXTM{;!gH9IuDY;3Vm;Ft%stvbE>bn=i|5>wWu`a^; zM(lFPJ82@4u&@;?`&|H1rz(bS00CEGG>VpP`X*&r1Em-PHK?yHqS@_6ie{!>W!w*z zj2XPEYz-w4eJONtiPrR2~9Fh_Via7^!w=33f{@C+U>Opk1`yp ziyS8mN%Z0|bkrGhh4~rck)x^nJ97mW+noD}9ke(dOBsy!r{a%J$JA%3a6>N4Gi~j& z1hBtzp~oflVTM=ihE*n?7zTX@3HI?Q!y6D!$u47I07e2g*{}0jIze3Sk4%NW=@l-$ zSr!p^s8|KiMV!;|F?%Hkco#D3wS+cCN_&d$wO#36BoHvxCQnl*9kaj^JKJrxXDs#m z+MZ7-M1Yno3Q6`dxFV9~EJQ+XcwW#&j|}YPm9rZE`9_{Dz?Tt!GY*W!Go7F{7N_%# zwuNBXlc}B7sg)laM>LCyC+$1k5%GQ|n)II21}@X@(`^AYeyXpc=ypflFR+ZgK*Uc1 zRF%}P_M_NP=8Be*2#M?^W&V^KMn%Ed6_hkqVhVc{u&hzw61OFAK;8 z^DD?V%Nn-%`zeFde*Rt9qKP2sulcq30|SY2an+(V{P8?NM*n#>KOW1xpkkIkN^~qK zvODqnfS*nk3R-;0-!quFFhZ67)w+};dlk*Iz9-?=v<#HS9O;gqh<+%QffB^(hAqi3 zY}B_ktws^I8RZr~~n zre7ugu5|#KBJkX-zA%%9nJeU%luyv}yu5+f`xKN9m+W)jn+p`(Kd|BulIdddZQ`dwEd7M4V~v3nQm$YUj`M zsdfw#m`-GTCP&$O0*O&1@IPQu)39N}Sn{7wJxeBuO=gd9ops^xy~E){Wf1$MnTr!^ zP>kHKyWWB$VSe#4XaVg%?TWvZeDk!HEljk>p|Gg%AC=2o)w10f*PT%p<3fXgy-T*g z_H!>-B4#H(QDteM37|6~Bxbt)LrB~sSZ&+g?$F=dgN)9;x+R!!gFqj!ktZ!RzwjKD z`AQlPVn*BpO$hj@YgBdjZ*HXyo{K4a z{cK@@E?`3$nTZDfVgHr7>LO(qbVv0ZSK}v~?zG^rVaf|i(CR@%n4j^NvcPovvL1n2 zXx07}YiR7S{lB!_-`8UlWbRzBS)qOU#B_tb5pF!U#6y~%E$UWPVTUZz?FPjVL8}5G z;OAwaJRYb?Uw$Zzcq&U^Io5~+5)8b<65U!EkUTor|v2x)sxzo3*H+tSfg1j%+rN3a+ zk7PtMw-fq}$RV#?^sTevW7I&)^q6;o#FZgS^7@05>eDd$CO4Iz)z}LNdRQKr-pN>mEA6u% z$t~SL-RNd27$u7ig|j+-O}iu4(pE}FZCwv~G{7JNwCO?|xAO1u$EL+Wy$WtMx_rp8 z&vRbe>HD#!FW29~qB+$%4q~Q0a$%-@@Uiikql0e0yn%npPTWjFd5+?5pq`K70|F{| zJ2xp4d*2)>I~-ws)k7iNOnPlQ(7OTB&gy5mpE|9~C0@eQe*f9({nmtY&ktKN*v{S6 zSiVMg=23skmR8}a({R+$N6jW4dEpaAO}HlYUA=<$bqk9B9yFp@5e1o#tTy-jM+68Kbvx?BSLd zJPYgy$bd=8vo3s5?CVu%I*O-5e%~OSH+)#UbQqYXcoWa8Kw@06S+j==G)vv!&~{SX zMd$NG*#rEI7O_-wGxEQHJDGpz?SZukl;HH+a1OcX1o<{Q^`6|u+aH_CBW11`k*`5& zP!ObESo}fl&hHA`z$*mRlD4}9kg@>rW>0j3RdtS?woXsVVAm;*t>;7)8x$|A*&lQQ z@&WsmJ77!*@CGKHe&8po=J^6N38r?^fH3dSFbBaBhYQwNQv$oMrdLXLD_C8|6P-?| z8jA{3?rQ!p*Hf`5a12J|2#g*~9`8V$}tN(#r<%y3$8ua1I z+G}mhV-^HKKQ;IZ%{k&BD*GZ6WmT4kH zjfRO1+N^Pk46Z`uPZ3xdcq@bgqM2bEOltNus35=NBuuvTTW(dD_fN296^^Cixdq&sm^AXUnhYzpA52vxm-TbW}IAR5Ho8&4qu5hfIl3w{FAK7Cg7YxY`?Q0Qw!T3el6NrTQ3%n48`90D+h8 zMTs9A=Z^ujcSa_oR2vdQH<(u2N=pG1TMpj_s*U=m{9sDuT5GHF+`MZJF8wz@NVQtXi#NCmYY9owK@D@%cb?Y5+;3rF#T`4t{v0nTEg4jFN5 z_n?uVUd?&qKrsy83*+cW)qv+dy-t1ZBR@L+EQ-5B^Ed}I;OQ~dr!H@o&Mo>_wl++> z?vH}S**gnK)oGkzu@dAs#(0>=Q2baCnR)#yO_lKLQ76Xpyf!A*p(H?7s$L76F5NB9 zpWZ;S0ruN!<-+|T+)!)#6+xH6ca{RViDR##Z$?k4P`n3%(Gaf5#yj*Z z%E3l(Bga_?;iOMp4CtbER=#$;;Dl$Rp0Df^0s6U@+*X9v(tc>c;xJq}4Ad#IcpPae z{_qq8NGUdNq6$X;{IW2+rheOOpY9hqp!)}c>qkunArbC+!$#E!GhPq9(C!FMQ= ztdrGi($p@{p}2yj4{n_6cw-?^&yWo2?Sz*Qc!(-^F%R&lW% zwTY(fcT_{^vOAyF*P?}s_L$DIIWx}km=pBhC?(bMQbM(gZl0{Cr|p+*6*hY$I7Dr7 zB=q>XyYmJhIrOwW3=o(~le?pi==GM+=#HM}eqB0%pRg+Wvw1;lk4pu5AoJpbk(jCefQWBM#h0JxLIQvn~WF*+g=N zAvGva!7Qis7}-SK{U&ATneu`5lFOF8;qJpzUhbOxj3liVuso8wp%|&egI-n1wGZ?E ziJzXCrxQ98^$FxoC}T-SK|cfzZs*0b0)Yf@Qgz4-{-E8m$661wd!(xEBO)8(Lu)Ve$+r;1$k#wf3k@U;`_YC|L?HwQaKZ)=x4vA zFy|K2)}ATTp!+P#41xxQI6^7B4(QbLxCzW0=UR@Dh*l9I>2l^4w(-hl?*TECc;^WaWxn<2zf`vi(9*ZB*9Uf?kYqbXY`&RVH4wc@1nu0yEbgdG5 zJHffNv0$zbb#xwJN{Sy7so_*SJbc9iy6IZL%>go#4W+Av&sPXbZq6kDFLrt;Q_Qmn zCWj}7W}oQh%hU#CV!P=ahOqX-4TCR6p+C8IaR&EU8bASSROgO`uJ1R=Mu;8 zO44DV^2-jVTazLy3LC@R#t>lF4^I5jnqmC?b~5w9d58J=sSAvNaUC_~T;(I!h2of{ z+aI&>BVV>D*R6%`g+>0eh!ge=^~mIf)B+_L_t`pUJA^`q8}yU>L58!V`i}utU1YzU zWgoR0d&Edvn8NxJNICUvab8C_&AHyCFM#w5v6|XxZIbK(78wX)MIa|cp&v@w7+Vy+ zxMU+9H3B}jb+*Hi(L|+2IPJ2zWO}G99)6g589RHf5}6i;+wVu^-Sy+Io0$w?%BK9Iw$!ex2D6s ziTC2UqDB&rDG>AhNvyq&GrzZyD1d^ALTC!KH$wcc5?w>=?w$YoSKHAUD7Mi*jP12d zHn&O5LiFK=A%7zKS?Zn4NOi|86m+lA%lw*ViCh5Ma#*EVQ1&e} zxl>W6j)R}l_w#vK`;>!zgNy!Ft7l%n^zPiNsXEtwyv}f_;;hbii|`=O243IooU59* zP2uj0_j*mo`T1)!(98^6nB#sMS1Q**9Lt&f_phG?U}RD=;Cx>4#u3~7uq~U6+W2Ok zR=lm_Dy)OWXs8J!%#)Az8NeR*6s8-tTm%t5Pf8f^K8i%E>c1$Sr&&u$q4f!ATNkya z^_7v@tkta!?QWfldJ)zmJ zeeY*KK3n*+^l&kZ&~Gc6opRWnZURZqPFWM;1n-&ko)ME5m!Zd*bg$7qUeU|Oc`Pz! zn)3UNAaf}|n~fj(mfzzNe<4km_N{~2pM3|%A5GH#(2i?f|Jhd|^@QjWHY5O3BIH&@Fp6CM>9 zJ9I3ZE$bO4=Tk5rriDBypmqMD-|GCqtgg5h|hkBL~_^7kTs?c(Qx{d~$ zjz~NHs%FcTtACb9`tQi5f01dj4vac|$k;u7Vi(*kEZys4F~#vOw>C@76P}P8$se>f z&s^`;I{Qri$8!bbT};X~zr{MizfZoVj&aRBEsw(EJ{}YjV$1q!=g*khye1HI$#Pxm z-pn%ErwBqP_>NJ1qUpKn>Hrv><9&MK*$^SMLJt$@nCx355z%hTMv-AlQu1zs37S{e^9(gjCMS1@3|QGn-QKz+oI}Q#7fhHysGm-hUA-^!Ax`X7!oMVf6)8nJj-ydsHTUYaLY`N#};(ltCsrhO-Ko z8GFU4BR-NdtDqjUd9&3C6oI$u25?h`UnZDeIKH|uzEiZ20fPzRRZ6A4Y>*K+M^wkw z6u;%+z)Vx`;W({{7Y_pqgaF|t7pLGnP8Ypheaf*W5U=BwxkN#}M`3}vI<%E;7J|Me zi+2Tq?1_T7VFZRKkYkcZZ9}Tj(1x1jwOZde=q^Tk!F@~HsD&S;6xKb@wnxY5Vo^QC z%q1?w?_lMT`TXr+P5&M@Y|Kdq_Iqgf;#AK8A-P}p&PfcWdZ33edig1R9Immk4lda#?!+Fc2;GS1#Qe>0S%nGl zi9~^dkp=|YE$qV1k?;0$uM?s-6Ko38U0*yn=?$VP+gS=zeezp&zM2SmaKT&% zh%Hn)3%yS$W3;vOf>yQ~0NvYb;#v2bYE+Htp~1~At*Op`3sRzEikU_V(nw;$`I*|8 zg$%kc(>6rVw$)pO#X^NN_=xz>RH`V)nNvti$&a-^kB6++c$!kLvk_$p4Y{Q?Eb()+ z0(Y`e-KzPMvlYY`F_4$*`}fM*`4&?!>& zE{>mi_`)w-pC7K$Iv~|)UrZ`B5n2_oK=#f{6``-N;o8umKd1J`O3@gNg#ccVpWp;K z%m*E?+pE!@OJeZ!Zw)J$kRVGLFXoDi(?-47jck$Ysk^dMo|6%9M}Cfu9V<^>J1XvN z67-AvuyGnjo)R!h@33Dr$x(o#gbpn(F5PNQwH-f&X#`@kgo49*+dV} z0zhUUMk zY`vBlFxvjTBXtdBs?GIp{EXSCh)sZdHDqy;PK0Ars(!=TY@|gX*fXm$8HK)!R!gd+ zjVq2$=8d2#|70l&RYJqumD4tW3av+vqyjgc2~glaBgY2)FhRk@`AnaioejM>WR}~j z7taV{VGt_KjE8B%`}cFJ(q<0W=*!DCqx4qAGXL1mY71YNb-&Gv*V$YCBKhaG3?boO zuwl)ONq~+z=+~wNvF0YLB-_i{p`MX&Y1SOr+@{~vXyxDwK=B`#w->A79%eaTR~P`% z3D&~e5&B#pAdC-EqiG=Q4ShoE$UhRne#BdQ+Y2^*r?{8D(GAP_)%ERGuAzhhXYR_k zo|gvWKPV~+#8$xP{%yr2=#21R@+vMR2@_}(Z1dKXc=IR4M^NR!V z+64q@xh%Uy3YpsY5#|V^$+guF;9d?gDph;W4m4DBg5tvOkLT|kK-nJgh<3(I6auG( zR}s8V8Y$l#ID}H}_`g^muBAVtr+wh|-*e`nnYz#WGm6oNT(`)Q5dQx+`Mw-00ay{% z%cR`f6qT5F^5bha#UOao7E^?dSo7C|$3GqSe>9h(23s?3#m}+m714bmv2J$v&qBP# zhVl*OH7cYQ6+pX-4`mnGWBsmN7n1yje z8G%Q#mQ1Wm!)9E53-wja%4Dw~&iOQO7D#IF+Z~9EwQCx*93RTpa9KUC z@b&JyFye^l1Bcn!DxRVgq{(zQ&y#x4PH0@MnNdR(APIK#re_Lx6^a+G=r&sgZjWen9w|6I_`PN zW$icKBOLCx!tq!q)GG>4H!~tO)}_t_mXtXs^L7QbsQKLPS2;$0W%XXLW6mA17 zLy2VX@D(-hQqx!PLqK8yr29^4j;}Sl=IKN1+HQ>UVQMLN8Pqf&=aM{Kigc9v`gKh- z3%NF;^kKBJ{Ba@fXJN}Qg2wY1-6WN`3Su?Xo>xmx#Wq|!VTN}mRp?f*=m+4_D&+en z9$8g*rP+yfp8eNPm6?Ua1r&{-RwNA*A+qpAE7#t00^@fqJ_+4+V^+*yLuK;Yht{^! z?HpdiC8Q^7C;U>(S_%_Z>u?c+V)R{{&k4= zBx)1Q?Vn1&abO>HSt=WCl5y1@!o~J7m`y;ta95CO-09xmA+UC1DqBxrRe}E># zq;4p|L8>DAwStjrzw{Nn`DliD^6pAk@zph6@mJ_t|q!l{dcepRF6S}hl3)dR8UiO!KXTf5X) zR6RfO11pHO0LqAwYm*~{SY2?DyfV*OiI32LL}oV0Q^jthE;DHhlyRZR2P=)5R)lAX zE?0_Wnril1SG3jt3mBkA(5}$OBqS~Rjw8G2r^mD0R)TFQ>+Ua?e8)H?0=yz;NK zoXWDf&Z*msD$o&;0Qt7fxh_vUHYcvm!m+$stA&Lx8vEI4@o^Hu}M$;G5~Wrk$QE5N>Yu+y6kzX1R3oUwwq-`(3TL z9kK!0pC$wj5WZhOp(3SMd8K$4AY=2C zQo;GMdl_!|GVihpk>&POilE2oU{&xbs0#H$EbIt^-yUUAkkE0vaF}Qtl{WdTj`uY* zS|mao8=)sZq8K@?j?7tOtj4V@d>&KtVm-}osNg`7&{*@Up8VfmJ8i>fnd~&W^jE5n zvziw;chNgJ1~)*gUk%;^`1qq16;s7=WZ2?~6}$c{UepJE3f_%SWqVX?eLjHHEHbvP zEH&Lp9zH}dut;-|IaYM^2H9s z8Jn_m<+h);F3|Q==tZJH;xI7ZVndK1k!Tj8QJ-gPTDW<8I>s3TA2|AhLZFj zfi;2RTV)eh$R@fSdh&gUNa$~uRjF$1;#29YsZ?Pb3If!@ik5@sk!P}0>Nn{7?8}_6 zN_21V*AYwWrrD~z`-0aPs51DUE=!cV)apufnTK3nxuAQSKkpf@z4=?8%wF+##Om$V z&0j9x>!BWR|Fmz!iXW1Nk^_~dHd1?z<#g=c!nnDL)vappN=%CU#{4P-IJr_tKlKz6&o=Va2)5FyT1o z7iQT9W>hGM-AQl9lAZ`jG&maiKpnJ1eu@#eSbdlQA-ZbmjXn=nCN+MngorJJ@mbiO zO`FPT5x!L0H_D15z-^RQ*(Nu0w6+XAe4!YD7(2dwOM51EPsXirOKJgvPr=pcTzGYJ-EQ9Dlx55wK^avoj%xZF^c% z)!x==ioa1@%absY$^xXEowSG%m%OxTW&A&I{k%JEKZy+!2k!SWUlPBYJ03cseu9lM zwAQ_MZaS==W$yDsh^yjWY0#SSxG%&AsxYq!QWG!B@|!aKs`AvvQnqvWOX?UzRT!7? zH&tm%UK!p0;=JH5+6(^uNySnOP=>@dP-yZF-Q)bVCbLfMe<5WOtk$L`)6%p%di6ds z2#=uWn+6s8WH&r(XXUbW>vcN7;;axR%j3-xG_2uF?xINo=^r*0?Wz99pKZwoQ4rMq zj)yU2XG?vi?Oll~@?WM|jLF!`BCY|4x5D&BV^#5u0aXOwgYvS%Rm#M+3VBAS-oZZI z%D~6EUZqQN?XZI!)BV!f{~^^?r;E5WWc_~^d#db6$`N_a6mi!AwwZI63L(}IVh=2lAC#W6H58_ql>}n2%NzQ(x>}?5&F?J1v*j3XX_Hy35R_wA^XG z2u~!fGQ9)7G8W~AuKidPi?5L$273cbw!uE0nN~!yTLWLRpQQ$1^?+WHe^5PN`;#|v z{J6GkH~uv}|48?bQK!7F!ma%8axWEK$BM&GVSI8*p3@^!nep|nql7iP!{bzhi)`4r zSs%*lw#d3Eo&PBJvD2sH5I~#?0{1ITH4)>|wa|g8Q^(7fk`CE|+gSmF%Zr;N=qyL0 zI$2CBoam~PgujQ3A~b(Ouf2^3u3inFUiFE$U+FIZ;2l7M;m?m<1H(^;PTp$^Md%2WP8a6pn~ z8JIj5?#l=2L|FGH4Q){35yp!kjlv!3gyG|i!07iQG`&Q~+HX0>2dKUEj$RmAPP|41>&8QH*1bDr zjOW})5av2UWvtar@wC$qg!D0rKEZ@BQbtZb+exb>Z)#2~bSfDYNwPF`95&XD*0aTd zF@89OueZy?$NZt1Vhq>$1q;)P^JOEe-Op9o#SnmgZ=+oldI=YPTu*9UXJV>{rFIeP$f#vKyISTL^*+we6%h+Nvw)3w?$hQ-21wDr04b zz%UX<38uEX7%5!^fW5a9c5$iwQul0TL9Nmm+h#ZVOHfmLylh4Gv%Smv8*OEAnvgta zf3ja@R^7iKz46$VT%^qQ`AXxJeiASJV z123BxQN)H=2QJ>L{XYwU1|ID{UKC?Yt?jL}vtDC53aD@G<4-i$Q1RW!g@j%bwl5E* zKY;I!h<-2QeE_?v%n6_6Pm_d;TES(ih_@sWuEItMRS&KzV=lG7 zf-DIMPz9q|(|!E%!NQ-Z1Gb;knv??HiMWg@p_09pq6o^XY1AK-1pI!2NQRq_eDc4k zA4Us&>+qT*sHPF+=pJCNbD_(bOAGrA?MQIj`usT+WUEr_IY1~cdIFlVX_kcax{qO*m79nP# zEs+eg<;u^;bP!CuU-YJvn`QMQ3E5cP%xXH3r4I=n1Mmt1?i^^U(r$#!jM(g?2d!f< z!+#~I&Eo|9MM;$F0BPq5yBkQs=NiXS7FFB7z*V zk*T&;6yqq|H;XJDeF&(3k}i3N+V>RGO@GeFAime<@iyP6b|h7`^Z03dn8}OqYLZ+IO~C6zj;A z;U*T~h|t}EpI7I-8V><$eq_oj6$x22so%V7J-`=BJUI3#Ij3PWlDM4~w};uKK0EB%(29B})09@l1H}WK_AQ~vx@?5l=mykma1kQ8k7sEYCW*ud z=(IvyN-vXB+aP<-L12e<)W`m@A9-clTc_TQvWD#iez5;c3WBuUJaT#5@-MBA2D9J3 zBV-H5j?2N-yaBp+XldgmOr-}IVP#iuh*q8qCLW}bq5 z26m)v|7*@T?XZPx%rDS`kvh$d|JCw}V)QeRzj0>PW_-2QAAETA*-VWHlKj(-T^O$x zbF*i1ZsU~Q@uOU!aM;qQ9U;MtJOrg%-N!!%8`=tVs`7YLPRM;CTaG&Cn(UBaE2#BG zheQZt>MIU!!smlqNeLjfyUC999pJCW={ZQyYsY-2y#D?KQ8>XB4kj*3yvQZ6*T`z5 zu9R_7G-BdCN+tpmf}|CP*N}{UuqJczO;THItxps@wnk~-$kFu_c8ojeav=V9;KuGQ zEiz zetG{9_@SV05YZX+d9gnuW5CQay;PB@L}XeTVSUMCVQoqz2}a?A{mLP9!9A0Y*WiMo z;`+6NBzfYBRzaPJ4#b;G6sXg%Q|TBl=1PE7-Ah9tj*>tC>%Z*n!g5@E>m&%mEO*9W z90lEd^&Wp2YgknOFdhJ;{CC&19vxM=KuW1UIt{r%-RBbNw*}>@Iadyp{5CArOKB83 z?RkI8>VbaT`U)-o_v?~`j(fC^$FzzpY&l&k$WeX@((K;4ucfa6uQV8S8TVEHJ&wYlCD<{P+ zBDo*(ynl3}4U8anQK|{wud(ATCKH`NNQSo8O@GPWru76UnLt&)-de|;fl<%Ksfv5I^(jye& z3wAow1{v68SdS=Ps~|Guum~^cY0wmh`))S z2mJc6uXDCQ5+0+t)ZU8aGIUGXhCBKJR^9nSPBPQ!xpxPme?|CUEL1(+=te;89&!)2I%N_;@c^Xf@4^F{_p+j0M#K@(#{BN!a0j~p_R43=@N#cKVj+UJN} zgdWI;8O{{l<>(8_7R{P)FPRDySF@NWQ_MChk!kYr6{2r22@drfAg#|gw?}mFJQmo5 zHN7hu$brUhM4`QyY+q;4=m5q`E+j+A$hpX&&b_|yn8m289<%L7A<*{g_PY(Pq~z*Z zJtt*L=D|ZDZSKmc7XE#TEYG{+J~w{ALTWS}!z`0T-GcLS=`;f<=WgE4sTL4i3p>pv zNN4>oyB8PcB??gJ1G(GVUuXh~@v=M5>BqI>tX-WXpXO=SJ;W{o$!mxqBQ$c8 zl*JpO6+-PO6%@84Y{(czhNwsJGGQyU-`u=J$aN@i;E}H=d}0NbD&hTaozKQR_j>Zz z93PM`c+&_G04BxLuaUt<;yCDs2ZFF**WJe$AV3FDng;J0Gzr$}@heL0I`==P?}*N6 zlNyd7eOe{~5z0F!W90nIop8nO7S2XO#~P3wXkO^0dJd|hA<|?X+9q*Onh!EM?_~IC znHz2`CGz?~xvno!S%WBa7~gQ?*Nf}i>0TlMSDYH;+N}i@16)5Px0YSbA#;*4wL!TN zKHia_Y}6@M791yp<~^Q1rL&y1dzTH~v37hcAF5+Ck|4X$DIWk z>7}0wlv91EadqcM(OR0DrONp#M+LLyRswAYZQoAi$&^pTjPQ1aJY53o7oUUhGB!`; zwe6s|QpJY8D|shZAeX!}A|VKW zkNlBg8Dpd)-4Ff3wHrQE2tnXq=M{y!9p9{{eC0z~!UMN+ zdm|jxyKNK0(NZiNZ&6^&1XKq;0RBPp9}JXh&lJ(K!xS&ogwRAX49s53Tc1pRI=TPY zhYW-)2BI@b3R^U!l`BoaWqL zBiB;G1|GNt`xKikks-~B)WU^u*AIBVm{F7WEHhlg@Al<;R1=RuDo~O#PnZ=}(Y85oX*!M)1Y#neLz0bIw1bdSn5Sd5z>4}szS|{ljp9_5j z;#bBh*@aL3Dfax784}Oh^j&$try{QthajI&UK2B6^n-6mCQFxNZZ#>0a(UJ>Za>Dk zkxE0CoH3=ucDsuiq`lL8FC$=T~a3flvrs@k~mF8SSyycoO zC6>E0@^m)ms}_5g+=@Bj&&W6p{UY~MdY2s!3Tg~16YA}PpM(`^M(~I|fA5i@KDkv+ z6ZXx<79H=V6CMU0^nvq?4)=QGwmQ#%Ltt3>fiBSr3qz@iQVAb=r02UxOnIdqDl}ro z*nX@Em7ilrG2Q6tp})U35cW>n>*b7{sPA_oaZ0=O##Y%l5=mr@~Kw15DqLcXunlB_Kz$=Coe^QaF3L45DolLY z>Cs>#bQq$YimeGO0@7Wj??9~2?8=ff(xm2WS;l#3N_y>5Q7{dW?^I9~VC9+zcI3ag z&;HD6@eI-OS7q3>OvI@;121Q~ZQo6j$kMO;-~o#mThs_wiEF&PJqRpM`qVCRtmBlo zG~mwBJlYgFruLF+rS}`9+Jw0rq@0Y}72>UOqB+ScC{aEpwSlAT8qg7bn=9c=dQL1j zIH-42Vi$v+jJ*06w)xc*-@TfWU`q|*T~Can=HsWu)IZshftxCu2?ELH9kBujp*pbs z@DGU|Ms&ZY3@y%z-YqC6ihFiS&cD`uH{8}@o!Sne`dnDZ;hqLr%P#6L#Iz;yq5M>4 zP+y=4Q-Mua=-PYPRM4Ml3eu*?1877l;!PJeJ$gNtw|1dJUi2P}TASG*f?XB9ZEb{L z0U}djWzgG#vxqU2u~0HQw*I}0O3C5P>qbW%Y)h;0Vv#+$UKTT|zRzJP9Pn};6^M*} zqobE*f}*~!3YF(UUwJLMHBC@n9Q*Ps>(jU>)gi%Ou`r=Is)xv^Pb1Qk&#sQF8J9f@ z)CEJ^2+vX3#Eyq)gd{k#mb=8$xB@ofaXP4caei%qjPq_7QsO*vyy#fbtk~%)F{r>F zA<)8+=$LHt9P!enG%LHs0f8yc?1H$$1Znc;tcvZgOhPt(kjaqBWPr5v!kih8@%ZCnQxpnV z$W03d%Jz~@UWmwHMTKnBj*hX2A-XD$5B!J8E#4dT{(yy-e3DZ>k_D&?lg#2hmX5%Z1=(Jwc}Bb8cGM0Q40>IMq2^cmO9v-8Uo1 zYk_goJ!%&e?5pajxr~T~ii|WpGw0{jw^5O^#oJ2|WNwpDDt`6Ho0dd*ReeNerRM1# zkJ3{yON%TUn=i`AX-pavJ7-&*1 zn;h}~_T2rKSJ`tTevwi3JC4ylAve%yhLeOPsE_l0ZzBj~G(+d4$7=3&x{+ov0=>i% zWMQzZinWD9Zv;hR3!$=JtvVhV>c#nux(amur~8&ZRx2{+d)98b{}DI?mL5i&-_h42 zyU*KwMfM2)@nbYh_y+P!4T}s;ejDKar|SQopS9LLDw7aYBh&%}1&*5r%Z>UT(d7$q z0+!CByUkk3=I}f}MXj>a{h1pgOHQ+xhL%;{wu-MItV=!Mp}g^r=6epki}-@SpLyyj zwI75kc>8IWa_9mOV0qV&P zDxbCyD$v_ZVgrC~W%QGJMx7ZQ)O(QdfByVpz6kRpXWQ+@1_o3s|J1~{yC0a}l~WMz z4ZZhLDFYzE8SUD~k5Z73H6v}*SThWHkzxL|zo>hUDQG_^bwnh1SqFvr zzxbV)emxd|cxv#r&+gK!U4pjDDQdB$V4U!Es8F{ndKF}{<5_k8`y<#}3bDq*q0h~o z@Fb!NZJ}$Yo4^6E-7?zI5|^XK6^N>}#vE8ZP<^hF^Yw#TcRI=HOk=c3oJ<}!QY0%*L^s_EUQn*cD!997eU)v!EJJ!$1 z-(%6!1>IKnzfBWhs1&HV6{M|uANsxu27r&2rs&Ihae__MXUZeztA(T~n=-rk1ED8Y zLho4Vz@K61Le1Eg20he!uR$Wp2SBqwoNSmwpknlDF;1x^3cbG#xK>tD27MdDaRmI3 z!N2SNj5r#jFEpk)%9Bchk2>C1#ny)-=>H*dC)AH~={?Of+`T)*Z_KR>ovZ6-ORQh! z=5~3Y07o=6HJ?ALdcpLEE`EO^N)3XSy@%AYOor# z?OSqC!980w{BH|iC3hsK>`Fa|Eba^!t0qY^Fqz~Bd2QIi#JrMlcasu#ntAeW*(QI& z!bB5Ll}6LzPYqtiYJ;)3=s{mT3ma4c9{H z<_61WObDLmvP}LN{sst|&!*K%-^Q$1Dy&p_>HvyjjG)?kz6O0AAv<_};26v8ZDwUF z7r3!ZUR?q~>Uq;W*xyjXK~oi>xABWX2@z7w5S44X!V$D+>L_4BKk(u2cbq?Fj4o52 z8cg&>f)1xA#+Il=@w0VcRAKa3A;_>WI`oMQc;D3>8V@tgtF4Dj!t?M!urIO-a#kB0 zppj-PaOk~Ct;e!7&kq)<1LO%K736Re*M?syntoDvC-20G>_2mN`aDr8pcGDcob9?165|H58 zgNzDh%E3goL^t;Ea>eM6EEVYA6CJDAxVjDEkO92=cYM)J$U+CAXe-FrP&{fsBhAq& zpqRAbbTFadCO@~~LWfAX;7$zInmmxSkC?#b2M3xCAr z@$>x_OEfPnUHl6rhy6P0V{SxzQSPt+_Q0P>_CZG1&&5>S*E-PT8%E_mfE1yQhwOub zrKS;wGh*M_CX6Lm-8an0{yPeV zWV|@Jfqe*C0y!fWg|x%KrY<{~E0Z)O@@A3Fwkl68;GlxazX3@3KPquz{C3H|()L(n z#Hc`#kXVLe2leNi2En#YO?Aipaln~;lXAY62PC5*z4HN1ROu8*QDWK5kGQr~ z4O37O(FU(i0T;1h3!afMbE6E%tOl?gFo-XXdCzP6Y5U%F!)tSXt|TOhMe`dAwfd@o zD-eJ-%jlf#sz{^6DZY4PE5znJle}gPC332oOEH@(b0pVoT@B4SR<8H+P%!s|mF+@W z8al*FG%!xg?Dzdc4s?15JM8biWLt$9t&e$eW28&dT>abt>FtzKVxa%Li~$Q4*dD3} z=LYpUzzhL+<8KxrJ4(qX=bs?90%imA1~r_aNqbKTtn+hpY^NK`wi)80LrwR5uL+u5 zd_hpSg%K-@m*_^gf@Ekm&s{*tPVP1zXVyf~iN6$wq6xL-n3@}9LV|Gav#$pw#__yN zL?^5kIXZ?o%9?%qPX$s1&BtiIYiNqI6ID!23EroRFWxe>GwB z3>XlIgU|=bexGws4DmkTpaU5iv+z>AM->*AFRJkVButs}L4lbz;Zuo1e!*@ikvrT2? zBxa8DFkb0;_76LVN1?#Lqntlxew}kf9sItY7VfpXC_oeQqCoy#l=!3@#Y*ZE>mcr8PhqX{z;Wh_ zi~@L6K^EJQKiP(1Q~Eq!;4mUZODMou_ukvxiPlO8s0eWnY%*(maBdk0Eca z9trd!i*1%?7nKCgaj>wcV*#+xy}{{Wi8z($aKVF!3s|ateo%;#GwOXnMs_=R_ktXY ztV(N-SV^cjXh@`ts9F@1^C_AH0G!g{ZdxcChDxOgICamn9o`kL_69zBnqU}}nawpTo z&P;S8ZW0QE4qqYPWm5oiT5%6QEh!Ev_GTdQM*H-pDft!lASwPPNwCyHNeH$%St|jC-LgY65u`HzUGz**oELBg6PUC;&hZOTj*s0XIudQX?pe=~^n0|CC$RjYRaL zWy7_&uDnSOA^#IM4AS}H;`$=D%(yBtDi!4%!EOBG+95g1A7z`T$Bb%zRPx{&Ta2EM z09|iLT9IC^@#+Y3{?yf0C|cc@`8xrMI5IZSis^WMdpF!8`C+n~6p)#EA^z|s_q>B5 z`1&GMdcRSQOw49Gn+$*b5CjWXVyLqfA^wMjhvag)*#pqb37_Rw4;FR*v=rQanz6z$ zdK2eYpP0fJ&4HeD`K_b~Q_B>zR)UDf)@JPY(!ydmpNMS&<_UsLsPYMc zx7h%XWADE$c%%@xFP{+n9fBy~wJmRqlB};u6D6D`9fhI+S@=?6AYL$;T4|Z02Ahv7 zK7KKW55ctu)B%wGI&&!k>ewIk7A$hh3?i9i>=9=G8TnCfGN-ld6$oq5$82og zl-s}2<9W%?<&g}miQ(4_zi%V=@jnS3i@RHZg>oH;AB=F5v2iB2k0*3UVLZ=I?VDsDM&6Ay17E#sMZ z3>!DqML{GVOXe<+bT0vQ;wdn1LW(XSHgfPxH*q8)kD-^$HR#sZ;$O1$@&gZw_Ig~V z>=J{dzx`%T&NjQyAMqyrXOMFPuG-h(12$sAk~JQL%w*q5B3s+Lbia>@1gHxEFu z-8^^_>U+N45i|VhMcBFkwYz%YfGwPGs2eSYF!ozXQ8cf~YJT=g@LUUC;1L#sdR=0FGl z9wXL~`z`s|H{A33>;~LY$ilMKkSJE~p{u4xEMzD%Mv+G}Y18MLHV3Za4*jc*uB?gFfimvC9hk^wkWIRbg{3icnI-Skv(i7p13PnF8c=suwei z1|(ifdm3T?1R&kCD!}*rHm0%GIH)1r?~oA>^i6kSpLV?*JoiD_Y+2HVJ*^E-ec>(DTiJj z0L}6oCZe|9sA)l{rle5K?q??Vq#Knj< z^17fd+JvT@CGB81RGyz zZ(*fKK+lALHP(OW2%%fs43Zl4@b|A+Rl<4uthbnI;%CJ*UYLMk_Pj<4L}a!Kax~1y zW|G~|xjspX;7j}FrkV&0oH=?2cMxDmUrHAjMB@ok*h1g?+*9|t{C<*sx&{#0(rKG2 zC*j14iRXQAtlRrDoZxET;8vxks$0*?L)inFFa*e#z*L_}$7Fec_Cp>=#N_$zbn%|K zw{iTp0JV7#*29vXV{N_Ye^}NL;$>wrca8G2GZ6SQ%ucHT+d;sbvH5P*9Db?`YIRZL zMiw02f=>0#65R;T*r>{YEn_99MtsjVsmr#4ljTN>?G^_*&OA|+sMMu~9dX-|Xfzh0Rd0=BAye@hXNyt&&ImVzRn4fzu z!7Q^Q?rvm(x7)5G1xPwF4u5nW6Dl2>1}VqqK@?i$+HSP;JYN!}ODJe{S(Z5D<&kuOHg66PnqO8>I$(k5d7oQUz7` z6qK}oNmdMGH+|CnD<_AnrH#Z1D0Y~GSa~f;eN5_+U;RDK`&e3K=8wH1hBFo%5p+#=aFRFJ{n@XRM?yLBh@j zyAa2`-zBqT5OV=%jpwz-Z{C4xf$y)W03=6?G$`*SYjOc>iWwu?(Sj-6eRBmeT(toZ za#8caEr?hBpZu|nbW0_LK#ieHqvo6p=emQ_kK?+|C2M7`k>-molkW$8M)&KU^~U;= z2V|_2Mg>g-^Zms~gz^}+LMD%}i_+i*q)(l-^0|SK#z)Pb{lcyP`d5mJ9}`TrA0&`P zu1S)K=Ssb@KHk3X(-w-f7=%~1UxlluC8CU} zyUoZ0nsk;PEIHX1v!^FGCtaKF(rS0ckAhs2z(w#{Bc%hOh_rJQ_juIr-e(n@2QMQOizuY@if4)eXh=claK5!_jUR8##X9K73z?iAzgze!|&k$)NEAL zsI6T?Z=pJMcw;E~bL3C&3gM84QT~ziNmnR}1r}iH1{F!RAf2PKP$Nl05S8#P&Km0iKNuI#H~7!gi77j z&1dR9IAVHc4k)GREgJ^h_Abgg(+(c{F612acuDIKl{%MObD-&RxMxL`e_;TcO|Hxa{9 z2qXVOMUi~D+?7aakKqyv=z%1m)YzvKpf-SSczgIKE6kMfwwNrxZ8)y9Fhv<5%-%xw zD%{+1_`G;Iw<09EK<%Hc;V#|aBMwTlS!Z7^n135Pn4)pfwnfe=;Mg*1(hxqD5^Y$a za};O4)n5%u#&mL+N&BmkM@d4@IRr7bAvKBG2UfTVPHxpF?|;9({hjmQ35e;S`eL3N z)pkK_R3(eBMj3o0i7vVtU+IaRoBCv{k^p0R{41r48M8t#G^Q1_ujB(qkRfxyc3f~R z9e?t@@>PXuACJ301|oCYd* z8crw*SmbBH|Dlwxv@HWPebQUyKs#?+Id5ONP6ECGD|Hh49%%TKQe!#f_pNy$v47PQ zz-=3gX9~(>#m-^Z4qKFv;W?0sB=y_5tW(E+{L0dAD-nfG&_*zav}xgAB5FzUx~6%= zczPAWeI4&JHw@ep{UYQ9pJ>O^tg~%ja?x$fdA;s~qL`{FhVC*+1-G^X+RD{3It4fa zWH?q8w2&7a1UxtWb$5O0($@yJ%dkre`3=2QSo#O6<*o4MKFuHbSSu!mM*q=d9CjYu zgSTYhbuKRQk|~z^k>gjq5ZfcHH|J?EM=|k5xky;Nf>K>fp=8@ck17R=EUT1O?aZ9I zHagEsjefq+3eY80hJA?g)N)aYQz=s!@v9C%p~yc7DTaHFs;bjUJd4koR$lQ1bE8my zTQTRguN>!E^8IlKJEFjKW2D;ZlN|(H&7Dr3OmKV!(kP{+0=r!Oc6fyAoO8o{Kb1sw z1l#YuLHd#3k|ZV(@F(^$c}Z^t_EOYr){94^_)3MU#QD}q0t~J!S1&~|E9%^jQRKOO zqg~+S>{Ga=T1$48El4>V^qa$gx+!0x)H%jSDbB0x`sKgweM`FHt=lq6vaH|B62&|a zt~5*(u`y6bH2S| zL`>W;$3y@h^^q0`X0fRcEy5sYWcshkHw<(3af25_w;Fk{Tv_DKI`!+%;4k+^NtqNS zmGv_GVa>yC4ew7%QztSop@1ss4Taug{~XJ|a#t_c)AbQjqXpL6iNtrex#gMl3QB-4 z@DhmnQV z7|plG#2I86E%a|~K`RDr>GD4tZ_?lY`U;~ma`2zbLJSDFoU~y5O7_X_o;wInV`TX% z$6BAhegIVP=7)=TBY$2bB+C;N{P;_vQMM>shC~AvluU6INki-+&HSb* zpine^Rz=X6C;*G2xE;69YtXtpSJU5&w3xy#Q)j6KeViu?u z0)<-l=xO?As+V{fHSiKAP=&yPbco|@>fF@iM3Ssusa)VFqx<`v#Oq>OSlDzcWl8Y5 z{XR)xV_Q0$MR1{enFJKfQTh+a@!7f14=X+92R-*Cr=*Sq9Qnl^Me2+G?2Oj=(s2(- zFCijFH$M&H$v*n{MMa>#Ph^W@h4fuCOmgV*{c0KC`NLrWnYyqs(o4G2B%;t`D`3_YQF# zcD0~Ja{NkRO_EeqT8?4;Vj#vBPGG?OAu-f6_y)Mp+kDczg*QUd?g^do##wE2qYPZZ ze?E=b;d5WT2ugN0!aIhe6MCexl@P&G`Id-(mML%RB6IfvMXWR=*vAMX1r{7r&awIz zy8JG0#>T`w|C_rYYBuv)LK5n^<}7`?-G6w)fx^A>$6uEkgCRHNX8|Vdi4^>so|HS= z$k#x>78hjt+C8BPD`+Vy`GxeR$Qy=f33v^=+(Ics1mnOmiGpb$P4BpEqUd{Pj48;3 z;f;PDgn1Zh4;XFrzd2?&f<`#dizasy5(G1#(SO~dX`$`T)gALY4?fXAkF%HTRzv9dcYdBG`@YN$aH2sqk&_n3kR_IU3KL9TIU3ZC+HB3RCfl31W*DoW4T4js| zKZ;pVF)PUbJLtopSmmSHKR5v)$uK1T%y#IgP_^jzv01}QalxK=~QiCHH-g5`>9jKM| z&3hR}c&4>b{La3zROYev)}6Rc@6^F8jW?jsRZxD@r-v~chDE!yF{>9G&3anK_CD?GF=lTE6h2tFO5}5hp zxOH{}u`(91vp9up{!HwNh9Aq7>TxxBvtZzg6zC-s3F(jrHMZ8WY7BSojhFc@D^VwM~?PsDt?&T4BT-0sX7Bk~h%JW`n&nxKMW`MB)zTLDAjRO)jO64eF^B28dwkwuQ* z)z~7;Ce}g2{r7aAg13xQvRBnI3>hi1qs(WtP7lFN+uz-b?whIHDBs|AUm;_q)1(;O z(0-ZgYE&D7W}J=9pH6Bt@^gXHh4ry4RmU+YNx-QKTYP7*epu!^^4Hu1S%5_}6lO6v zts{&IUps8X13jN4444nkJDQI=zTL~oRza-eHr{w}a`-rCwed#cSwgsGMB;Tf!n-6Y zTe~MIv=)Pp08?Swz#KpFxbDY)*6Vl7V#+uz`d~uzyiK^g;^Dt#e99{-_A~=MU35Xi z+!N(E!4lcb>6adV?YAsVBOeO$ltsox3cMhuenf>>|qfV)WJmvuw>xyFo^~ z^S)$0v63iae&-k4&g<^Wiyo2JzOXXx9x8L&U1RV881s@Jx*8i2+uZLeAHHQ>N>ngo z&)@-~(96ZZFNlzNMjs;0Y#E5!xo%h@m853qE&B&jV55W?KnIc)i%L2`9$7qG{~Q!T zCK4v@F7s$X%N35<3?KG1Fv+qOni`tb-?H6jr%FZ8(cc28`4XwHpljmknD%> z$m+!{;~xFp^~e-fx)Ym%V1c^bx_POrIOI~Im-H=W55^UF0OgSxr$|x0&(4M zj&gj0Y^CK6WWBSR+T@`SJE5kAxN}Z6-f7HL2hf^z!vwZ>(6i?{mrjr+mDC4H)e906 z%Gll$SwkwHGr>0INxUBV@w2lR(-MAygOh9ACvZE9Ae~~Fgb_M)Zp@V0h}mk9+T~_I zOuSrZG}e%a_;-8$QNxq_&Bx14u5HWOriRlEQTWsAO|qd5Ii_8i=$Aif%tl0X(|_#NN)Q0U4ZiYw|pVd$zT zBzu=c!!nnrVA1=V+^4J%iDV?%D^w-T24vy3R1D&U=Ekc80N+~-gwf`X^zb+x7_eAb z`V=J6k>KeuB}L_~*()yXxB|JlZpiVHE`USO=kPvSL0JsWGO9w z%iNO>J*M<(Nx|jYZTK}c#sNuz8q!tiK*CT!j3UEmK|FP20UKxR3u%BT5FFkw`R61_ zWcm#Ar{Yo?*PhHb6MMaNV*TCPY^}#Q0fa}_HII{o`xx8x`5v$LGuL#sW<>5Sv8yuj zcvLTaT;vXa-!N4dVT#1J-xT)h{|Urqr|aa``;FCC+O2|EUUkm9%_q`q25&jL8<)d?f za*7b!7aveA3)eb)#3p4b*e&sP)rc%$Qzdezi-Kb2K&7x;ZrcDucYckbie}cvNxM9b ze>)EcnJwQ|S|oR7)>gSQw^~YlgedHjsMv{{*l1qM`?#?uk=aa#9H5h8=8Ys+l)+@( zpZqb3ay0YhHx9+r`|nn%D-$BKhR%4?#`xnMLD6IV|@%}%)UtD`} z3-v=RW-aG%2xcQf7ilu*!RZPW@#;d1*Wg|{O`A1O3cQ@}%$?5O$lJxv z8WBZ1Xlz!sOmM+SN#*(|Nj*IOvd%mbMQ>+XZ%Z&mCB%ftmDW#3O$lRK6uDY+zse=b zANP^+{fm1hd)$>27x^>KMb1W1>_#!59pUPM`1f;%itrxc?L<+{U8#K4e@XX7H$}o? znlfJ0EvR+eggYZmf^y#&j_R!cTE#RoU*{zO>n=a_{6%rB=m;#6`nc9t#TYk>oG@KM z8=7JMhfVieTIdS=?}HZV+7E|Qb2)Y#=$#Ju!8hNa%vFYFy&@th0_4R1AXwQ9B1DgE z@Q-ah1?GDPgs>G0Nr;~tm>4GVfYpgsfBN_ z|57U*z1hrlZrs)lzdAx-?Ta}=^RAlr@a(@--@gurjM30x!dR*NM(=g#S9W@PJAd!N zCKPI<`FkhGC=H`PUO~8_M!D2~sQq~8LX_LT`MSPYd&z`hluD&W)_@5k+Lc1Nx-^b` zRo36qdH~e{m=@Zbc?^9fY7(wXMR-q_%d&LkYV82;4)%9E8Bpf^5SukY% zWq4bq#rZ)1K0eHgLmp4yqFxEA*eO24d-ldPE?gf;flP?%f8G#Rm+P@+{9N;M>U~F= z5de*b0Di##gGczgBOVpzDA&kWoNXM2MfBqxe1O>T_R&|KGHOp_jmt4(K_9=B1-cPD zSv?qhE~%3{3E84GY$Vzz4Ogw%>F8*BdH6cUPA*yOV3_!y)kW?+rB}S($uT5l=b-Kw z@i7@l0h1sSRJHK-eBN6^M=9`)f=o&Ly!uhcJ_E-WR*2 za-{&KeanKaa~^G3dWT(cJb4mM$P3|Ftb8Pjyz$*Y%?X)kOMf3SQo2xLFTvm6x->;B+++MLpy}a zW6U9*e{YR}rIve_5=KqrB&wsW_N&Bjp-$i`TDZGl_Om8hp#NOuW527Tu#45Zxl<>iliXee!E`bkn=Md=@)>Fljgkb0 zj*_g47Bi?0%18B400F$Nwf1}RV&f+Kqj~O-TUaCWl5b5z0&Y3c={}j>h)vm%|mSm<#WviPSw~CDD#aD;1Tb1Fsn76QSf3!{r$; zyEs;8vGKHl!zRQkXfUw#fnTqN3FWzmV4}Xg2E?+WLAWdbK3JyN_r340a&$U|uSZ(h zcc@`!-TPpR!N!6i?akYIQvR%O;rZIcwj~!*88( z1^URNWWR3vVqlbXdO?V9>Vm_g*)2lnE4KBu0U^vi58Fq2iNwYGbIIu`yh}sKH0_yc z#Gz(OpPz2jkVMNA;zedXt>iKa?>Otd!sd1J3om_JqWk7MLbA2gq$Ou(zyF#m-~bXi8BZ3IA)&un5;Ec9+JjxRqh zpp3gsj2+=Ey|l!9D{SIg3174<>@gIZ&1KsIuE{iNUKM9;9b@)me5Oj%zpY?X@7PCS zs@5~B4ckXuX(W6(-fd0nZIytkexPFy2eoP$`nwyf;hbq1xN8XZ4!YF6nzK4t$Bb)t zJ6`_L&VQ`#eX2fn&AZNNos4c7it_BP1Hm1;`#;M>89i2I^Wy2?)BdperbwN*>)4LL zY5lb+;M;FGnR6Z`y+595HWKp!>j7maBttN~vOM~~d9 zv!C@y0vG%_dsK(lji(#$}vVI z9#JlITe+xhtgpTp&X!U6cV+S|vd!d3!{D98-wFzf^g>2LR2`S$zPkOtW6&^9iO0Fj z5mL90=Fi?sf?gF^%o*nUyqm-J*4cb*9_KO-ip`!{h!W|#_OZeIH|ZMfVFBnHcr9T! z`J1o|IgV9%j*F}T|9&t6aYaUSyB(|V?MG&`#uI4#Uat2>&iyZGsA$XqSstagBB}49 zea`#Z8Wyyye|)8pv9^r73`r_~M4G?63_#lN3!j7UNu%_uR}O_MSPbk=j4UM8HCOE6 zDPx4UxF{<)$zn!{iOVeouR-`ltN7$i;tWFf4%C#qkocFQRNYq0x80^y_c_>_ghghb zBX0knphb*GIv@-F<>NK2c&P2LYFN=-Hki2e@dOLEd8@0526&Yl1&xTq&G9RWMJjkl zdrqgHLSPm2g@5Pl)_jUyrFR1rTvV5k~i@JSEFq;b%(EGU&*|lPyf>*oAi>{h7Qxi8_9w6>cg-NLY1|9 zAzsURu)-AIag7gg$k-5afU?EFQ4gd%6?X7gu`Z9;G|1yf6gm9pG`YvIGfmxYk=h66h0_cSZ za9AcP1D~-5=5WU8M@q-(O|hEcuO_Z5!d>8{h$=dZeOs%0PFT*MTf=c8v6(bKQJ7uM zv-Q4$=TG0H<7tRom~Ke?_g?M6`Xa4~x5yx_t(WxMI|z{(;(^T)l**3887pv<-;Q~S z`WLR8Rd-vver@=+XP5@cJ#@S`GZSN*R1SAWAVkLPFhlW1uKr73*H%Q!cfL46ibzJP zB7c3ANpJWNY>AWG&*$5=8SuE^@=wy6YVMv(M56&=7c5YeK@bV6=A zrSCg-SSrqXZhOP+X6sX>7mA|)X)#9qCy31eWvv9Sr zzJ}P7w%L;wa~ckjb!^muXpjG!6aEq`&nXnUf546eU45=h5qA3N+eJw0LFm#zk89?U z!zWhz)bQ<34UA3Rw+1^;Vp!FUGMRG@F)aF;=m^H27rg<3KCwjaqO*1#5Xm zT}N{|=>fm_Yl77^L*+79PH5{)R>Xu>*`77d3tw(IDKQ8y&BwbSl^6u0c}ruvjG0x& zNVfHqzo;_%)#^)HvXYSZAu3YF&yNsd4SFhUxgo4>#^UzY*sc{eLEUK$yqIs|)#y!A zjK#4Xo0nv~)#2%s3(abYeGPs`Ng`=F)s3jc+)ASGI$^0`1K+zpmj6j^7Hd|qeNiSb zjZ-d@Ac3qw%@~nH5$qCU;}yzaNRQQ#$jU|!MXIG8B<8$7i#?`b9~ezhg&NI0VZ^~; z6hrLgA81wvnyk`zv2iv`xb`bP}&M9FTnj!N>e=W88jdefV5e zw$~c-!~VH_+P^MdAR4!mv1qq7GDfdXE&6vmV~0}4 zELOY#H=#GD2jUi4C@|IxYyiu*Kn^6y_ua(|Ca=Y8JwwRSN7nphKm2xT8 zEF8MDc!B`?8FX%k)@l6E{^R_Ggf5%1-R-s6waDtrh_j7NFSXr5@q2$i? z;XwR0+ppeAxKb25#O>b#0kT-&;UsGhAmlg<;beE2V@;}Zm=m$1XENfUH^;2IWI-{e z`+WmrnNJZ3Dxft%tlULT9vCZHo{MGkUk1s9?lzs}{6yO}rvs2uWH#N&4|-%!G|*^q zLvVq%Y7Cue@=xg#@a1{ESMEej8^5pH z3aTcu7;m97KT7mOfctK}0@m2CMDY*0!Dhc}N8G#w*q5V3$XN1$^2q>r7*+*B6;|Y< zU;l6kZ4<&*IQYfmDCdXCgDS`mVEEmV@j47V{_Lqd?TyFk@cA56D*%YY$g(w*IHI-^ zlUyKpWUd>)cS@yOo)s8?N#`osp%iLjG9;KBkq_aDiqgpW;!HD7;{VopyvQCqyBik76Xz3H_Bix)BXajpT*O5f4 zHJ9d<=lj<#ijdC%r1Qg}yKs$%!_={Q$$ns<$7jJWYQ-@bCc1tYn~a+upZ?O<;Kd}7 zCW2&-jnRTkM0nS-^dU;2nnChAyN=Kz9R_?ntgruT>dM2Re80CepG=lAqoOQh28GCy zr6lWM?2+tCmI&FhC2J@o%TQ!Slr=javhTa>`;slPXD3_uJ+JzGuaAFRm&?5K&b-fa zp8MSAock7@ni_ZNRd&9!#Wr}Hp>pI89ox2*yA3>IQ)is+Bopl__YemSg|zFYu{fr{ z$y;;cQ!gmit_4k!derNb`!3ifxFlMHT=Q(6!fqnM#`8?&1m8${=%`VU#{JoT4%Bo3 zuwH7uBI}nB>IDwtiK=t8Z$m|D>+buR?bCmM^m1%x zM;cZZA)fzqpWdue5weQ>EAcI12l|%IE{cK>AXnmWGMTIyp)%>Xo?5)S^Ox6l&CO%Roi-1 z{pC{^b3SKV)w9WR&BjpHY8<&0^DZl6<_+B5K+^pa&*V-YibYCtYH!$l)xvDwOq5NH z?C`<~?kzZHR|XNH_@&7{Ag=Xuw;Tbf+bn33{oC7SqwVJ|r_c!OTSVuSPkCGpC1Jr- zR6gr^g!}6-rTymZF8x$>94In`NlAE2s^?m_b?Dhue6fx0Uhi?6a*8@r0P5zZOO@&l zBVa6okv6q^JGqB@xi#eENR5(~sHZlr^B?K+dTd&M?uJ=rNUlS}vxg;mN*+o>e4nbx zZ>Kw$%RG+=Nbu8UkqtuWHTTr-OaxNQXjRHAv9lIcs>Wnl2D9ciPIBkpah?Fli#Chu zMbn^==GpW(oi`jyt_TNg340Br)x5;SXC?YU`xgy#|8h%`TWKIyHQ#{3XK8||HjA!X zSHG5#L88yaT5XnR7Z*%ItUYGlX$=G#H}EDOR+EX{+tm2IKd_qfJ1FGm`%2Z@-08;4 zQQUe~Z_EzIL{ptm_Z8GqW)PFi#Fu%NkvRP>-dKkx+9pBQ@O1pITHXS57F7Kd{x-Bg z!R6LM)hxj}J(=C2dsH3Yr>uv(^*(xSh#3`yyjq~}kK$05NSWkDf%DRU)u_-j8`QBo zS%id*K-$IAxbCE91-!{C&(Nd&VVLuDtckvu+%CpG@71~kJWvVbSjR63vw#05%lNgSY?U2d&l)4;JW4B^K*)F|lb`NY6QSXY&Gan3@!;VrA>>7$!-h)xrip5^a zR@r$Q|6v!gaxZjq>K zOnnt*D|Fc2{QCOKswLf7fiG^r!b_7&IohgDIhx=i4rC>)HLkCQ!O<^0S^NS&h9`;I zGe2?aX9NAKb`yrFl{(}sZZf30sW0oo;lsgc#@Qo=8xhf5|JZ#*v7>7;Bg1Y@1@6Zi zhb%aehVkY~is9wVBD^q+tui=y7%f2Vn|GdoPS`6i>|Im|xB6e}-3AaI&29s#`GUaMjgh34+na*NPX_+TrgK!(sp zddemw^c}*_@}Yl2!33B5#~tjOaOTddc!BpaoaShJ?3oU~-Wt{ylB9T@7Zl-3)k9_Y zr}z{!7V4R1lUkuUtQOe#GT=UcpITJRwL+Q|dr!=0r*yF2&P?17wc>y~B~MWjcXoj> z=%JQ`@rc$$BlC~@@%tKrSj3X{ZQgbz^?9*aW!kYCt4AS9E~^N)m_cU+*xYt&mSbhf zrSMw5!ZTPlKAASE7qWT88|7=YA`0rpfj?;h2$`|8wD#%RgbVtfcRHV)zY9NkH=NU*yociLxEZBz);`DdG(W0^~v#lk<$tjW#J z7vNvTHo$OM=R^G1UZarLeI3Ir*(9Wv= zBj-*{nI-(_cjs~W;=Y66U!;lsSOs^IXHFY`%4&Zh?NM6$ZkWssON{&oZROfaE77TT zo{FuWMRdY>v)n2p6EDXoti4y2!K~=X_QyV?Vd13$4gpjVHBg}-qjC>^2G>BR0JPdq z^%I1$TeeM@!|&GxuQP6PpP+F7!VsjOzlp=(71f)AT3%QXJ=@uzwGBT0D9M3(K4mUm z64T49;ntUsHMCW`4f1Eos=2FOO2c$~!fq{Gr#`f1-c_8s9(11L73L4T$HDf4tJ#r( z^~zGD)5>LL;Y$o=Pvbp*SwiC2=>gB0(E%F5#>1*Gli)lROflVrLqZWeY!PZ<(e=D~ zSvk)8Xpd&>I*-B$p5=ynth!HFV;!3dL;+T;X13GM%G5;O+AU!A!#xaF=)U7`X==8>DiPkjo znBqN#T%jUrNS<^I%ERVggJVpWsW4pEMPZCxrB{X-30M@uBIa^f$;)w#0M zv#XBwW$E%P*#>ejvRqOL`4%F-0I8kT5n47-qgVLEve0&HM07!3HU0DfX!fqAxO@R* zjA8TFy;gU;azwOejiP$Z*JtI=-fR~p1@C)skj1f+@=CB48O(>C#xPBfxlR~N->`!3 zro8qU^UYZW#pN$A?jG$<6OVVpjz90e9Hj{4`>7vvUmYpy6{{k)_hXSz?X@#zQ#59! zFNgNCEn5~UyU7NQ%&U@8syOYT zjCZb9c%%=aeMig6H$SXqhQ`ZO2j)7A*yrK*dWe_KzG*8%okpq&QcEFgwE)Q^Sld{2 zd{8CK>W7KpWp5Fv0$E7xgk8~WY$W`3O!E~bwn2M>isU}@y_kbu4J(M4(uh~rhvOLD z5NwTQepR17-)F6Ice`(dq?=E4v+X8*saQ6YxDz`)W^W3rtbZ9a)XR~B;ju^(rZq&n z5`>-Is3jiah+@Zkox)2C8BSTq8c47k(Ejev*Nw475`_gZIbPct*6#(8)lm2Fl%I&- z8Bk6*sG9gal5?oL327bf0g+Y$mbaSF>x`_qeQ0mpUMTwyK9Oj80F1(NkQH;{$9w&W z<*(#CWM6#YI2*9?(Xr}q*-p6QpesO3Bh}Sq+-*+b2r9Y^ybq(*x0l?$XSeliDXVBe zEvZpTN;HHYYUY%cwtmhIEuJUmU`jRw(t*Oqs8`8q`H0vtVhC?>pJ9v+HLXxYp**qn z6Tk2^&}DD*4lcIUCH zxFHyWqvWG&>og3b1$naC z(}({?S7b8v786JTEBj&b__sXR+U)jN;>-N%KTV!zo)xC;XY>~~k0We&z`q}RQ z9sMKwwj?v7LT5_=!9}RfSt-0~(RZ&va^s=U56H;HfS59? zD%b-qqP9&;%)(eQyyMx4)Y{X{IhB3jj&l3O6o}?v6b1G`ZlNBqy}of8lJEq=Vkg*C zNq?W6iuk<-Ci4e!=Vwxgw&{tJ%ioFy)i!`YU_WF!@|h|d@f|3z6|f?(oMTHhWT?zu zKCV_V;AX95AQ8iA?1Yjx6?@|`;CE8+qzEBNyOcHoWUa?Ot(J^Ffz`^8zl14pSs+B} zvEQ&yY0XscOzUG?oK@XNdZXe74|5DKr$E2TwTO+oz#c}w)V#!_-74zzy7ZZu{wo~B zi)SgBeJS@taUA@C@UMC)dos*HlfS*n1ZNl!tPS)}NXFjkk(9b$=>g6hRmPMiHX5G5 zHM+0yS7v^PRVYYq2xRvwxSDr!A#c{jf_1NhWvx{G`G~cP=2u=#@Yb}Ig=`QEF+31{ zDUtF?B~ef6x*gi~jW$AXStV%&+<(8#o!H~8RPl;91_-_vsw?Ch-mz#P-0B$7XcxtEcP%c{ox2Ws z_d+ip?wjgipK3p%Xmj^+(!;wrObP!-p5|TbbjGKT_1uq3@EL0EtW;hQwK)_EF9#BK z>8E8%Taugu(LFM>tFB&iuIUdlC9kHT4NObjlRVY6B4|P6ilh{UN|3WabR(*1 zQwbtEXkf2@WOJ{`jqq1I))O0Xo;f2)wr56OF@q3)ehUYfje3^qYsursL38r%B~C&KLnLO_>-4*WClq`YQIi4HlC4oaMG_L0oY>b;<7?5+IN(l8R9 zN&bS>H^Y}h^?xrLK%E-D_)M2N<3sQzCR)07rR!%|zFu|(PmLOv@m&G~u|*L1A5?t? zQ*Cfg?l3d9Wac@jrRrwIfYWFvZN@5#?|o_%zFf@?Ad?}YD*iGQ`=FrRk{xzFYrJVC`e@Ff?!=Tmey`1f0BHvf13Zu}eO%mh4B)AN3EL_FQSGWn zKU4>JKYo9rmpKDew^&uO1|8eD9Wv^=iw@3mtLa;`;d9GP&|8|-NOT?l>~Z_* z)k8>pAn^gC_wRL=M~i3xTikcm_!a+tc|@`oK*iNiyKiI9!PF!*9K(9MuARM>j_V^oA2(8w1q8hxrbhWfsPRYzRCp}yno zuwhUa$mFqB^Uew5QyW6gVWE2qb3f$kp!e~Px3KW2y|g2k;_chr7#nI|FW29JojR8c zfu8?=o{h}+rV5MLF3dFtOn`!qpl=IQ4Sue2c2nm#xN&2+)2mQcf&;y`n8$d>rO&jX z#6{1fu1M8uBk~(k)onrh?7=W-usO2RV|5cm25%-K6>`SsAn)4)NW)217ZS$N2x|z# zaV8XDS>MfUID_TV@8OhKQ}6k7j0dKU8KRMj8$MaNCm|xx3RAgp5_U;=yO@J0ig+L~ zk0rNkgx?;m*LPY1(l=*?hE2v}2U(ia_&{u}`ZO{AXdynksb0ny_JC{nvq+t>8Uu%hGQU9MO zoF|3T`Z|N=*t=87%hFraMNQ&I>x%xN#26%fNrcdk8_lu7A5LQFcq(p;-tlb) zf47w{%Nk^De#JBvfRA=yVbTYfU&eMI1EKjr!3YR%^y8+#2cT?iOBYf&Zzturf|d$R zlfYdnWKEV{k!C$4_+|CI6?r)gdsKVR|3Uw|ZWl>wZ-`Xf^L01$}<)Gtowd zA(Hc##=-C-RobWNDct1Zd|89vFfaH)IyClg3RiD+;q|dBo7;dM}&UvlOrT48s9FJWvdXG8@sJ^zfXkz>?1 zm*ZowoIfW9JdZM1JKowRnXvYB&Co~T&@;)30w0b6w&s#QZy;=0`ReY?opvS=6Z4xy z>nkG54Q~V!CcZ%k!t7nakhTp8R@$ z5kwb@mri1TW96UV9|uY=xfUqa{Qurpbtbs4Ti25|Z~yO3p=e17KEmAt=9mxvy)W~Z b)yJo(tn%3s7HJwuz>mC)3N~Bnq3{0zSMkiv literal 0 HcmV?d00001 diff --git a/doc/source/jobs.rst b/doc/source/jobs.rst index 4c482b51..63479279 100644 --- a/doc/source/jobs.rst +++ b/doc/source/jobs.rst @@ -45,6 +45,13 @@ Jobboards service that uses TaskFlow to select a jobboard implementation that fits their setup (and there intended usage) best. +High level architecture +======================= + +.. image:: img/jobboard.png + :height: 350px + :align: right + Features ======== From 49d7a51c5e8c0fe09462340286cb44a4ecead2a9 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 21 Nov 2014 19:18:23 -0800 Subject: [PATCH 104/240] Tweaks to setup.cfg Add in 3.4 classifier (and adjust spacing of the classifier section) since 3.4 support will be maintained ongoing in the future. Adjusts the keywords to be a smaller list of comma separated values that summarize what taskflow is all about. Adds python2.6 as a minimum version so that it is well understood (it was already implied) what the minimum python version we currently support is. Hopefully this adjustment fixes bug 1323731 Change-Id: Ia20120b3e4064fb315ddcf018588f3af4e503d1e --- setup.cfg | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/setup.cfg b/setup.cfg index dadce717..660c2822 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,26 +6,24 @@ description-file = author = Taskflow Developers author-email = taskflow-dev@lists.launchpad.net home-page = https://launchpad.net/taskflow -keywords = reliable recoverable execution - tasks flows workflows jobs - persistence states - asynchronous parallel threads - dataflow openstack +keywords = reliable,tasks,execution,parallel,dataflow,workflows,distributed +requires-python = >=2.6 classifier = - Development Status :: 4 - Beta - Environment :: OpenStack - Intended Audience :: Developers - Intended Audience :: Information Technology - License :: OSI Approved :: Apache Software License - Operating System :: POSIX :: Linux - Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.6 - Programming Language :: Python :: 2.7 - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.3 - Topic :: Software Development :: Libraries - Topic :: System :: Distributed Computing + Development Status :: 4 - Beta + Environment :: OpenStack + Intended Audience :: Developers + Intended Audience :: Information Technology + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.6 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.3 + Programming Language :: Python :: 3.4 + Topic :: Software Development :: Libraries + Topic :: System :: Distributed Computing [global] setup-hooks = From 148723b0fec0feb6bc598f8198974fe2356e22be Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 19 Oct 2014 20:21:32 -0700 Subject: [PATCH 105/240] Use the deprecation utility module instead of warnings To ensure we are consistent with our usage of the warnings module always use it through the deprecation utility instead of accessing the warnings module directly. This adds on a new deprecation utility module decorator that can be used to notify users about function kwarg renaming (of which the 'engine_conf' -> 'engine' rename/change is one example of) and adjusts the messaging formatting functions to accomodate for its introduction. Change-Id: Ieb987ef9e1e4515caa676a007ad00590d354132e --- taskflow/engines/helpers.py | 13 +++--- taskflow/utils/deprecation.py | 85 +++++++++++++++++++++++++++-------- 2 files changed, 75 insertions(+), 23 deletions(-) diff --git a/taskflow/engines/helpers.py b/taskflow/engines/helpers.py index caf7ec13..5877ee41 100644 --- a/taskflow/engines/helpers.py +++ b/taskflow/engines/helpers.py @@ -15,7 +15,6 @@ # under the License. import contextlib -import warnings from oslo.utils import importutils import six @@ -23,6 +22,7 @@ import stevedore.driver from taskflow import exceptions as exc from taskflow.persistence import backends as p_backends +from taskflow.utils import deprecation from taskflow.utils import misc from taskflow.utils import persistence_utils as p_utils from taskflow.utils import reflection @@ -35,16 +35,19 @@ ENGINES_NAMESPACE = 'taskflow.engines' ENGINE_DEFAULT = 'default' +@deprecation.renamed_kwarg('engine_conf', 'engine', + version="0.6", removal_version="?", + # This is set to none since this function is called + # from 2 other functions in this module, both of + # which have different stack levels, possibly we + # can fix this in the future... + stacklevel=None) def _extract_engine(**kwargs): """Extracts the engine kind and any associated options.""" options = {} kind = kwargs.pop('engine', None) engine_conf = kwargs.pop('engine_conf', None) if engine_conf is not None: - warnings.warn("Using the 'engine_conf' argument is" - " deprecated and will be removed in a future version," - " please use the 'engine' argument instead.", - DeprecationWarning) if isinstance(engine_conf, six.string_types): kind = engine_conf else: diff --git a/taskflow/utils/deprecation.py b/taskflow/utils/deprecation.py index 7e8e60bc..a5c7148f 100644 --- a/taskflow/utils/deprecation.py +++ b/taskflow/utils/deprecation.py @@ -21,10 +21,34 @@ import six from taskflow.utils import reflection +_CLASS_MOVED_PREFIX_TPL = "Class '%s' has moved to '%s'" +_KIND_MOVED_PREFIX_TPL = "%s '%s' has moved to '%s'" +_KWARG_MOVED_POSTFIX_TPL = ", please use the '%s' argument instead" +_KWARG_MOVED_PREFIX_TPL = "Using the '%s' argument is deprecated" -def deprecation(message, stacklevel=2): - """Warns about some type of deprecation that has been made.""" - warnings.warn(message, category=DeprecationWarning, stacklevel=stacklevel) + +def deprecation(message, stacklevel=None): + """Warns about some type of deprecation that has been (or will be) made. + + This helper function makes it easier to interact with the warnings module + by standardizing the arguments that the warning function recieves so that + it is easier to use. + + This should be used to emit warnings to users (users can easily turn these + warnings off/on, see https://docs.python.org/2/library/warnings.html + as they see fit so that the messages do not fill up the users logs with + warnings that they do not wish to see in production) about functions, + methods, attributes or other code that is deprecated and will be removed + in a future release (this is done using these warnings to avoid breaking + existing users of those functions, methods, code; which a library should + avoid doing by always giving at *least* N + 1 release for users to address + the deprecation warnings). + """ + if stacklevel is None: + warnings.warn(message, category=DeprecationWarning) + else: + warnings.warn(message, + category=DeprecationWarning, stacklevel=stacklevel) # Helper accessors for the moved proxy (since it will not have easy access @@ -66,18 +90,18 @@ class MovedClassProxy(object): pass def __instancecheck__(self, instance): - deprecation( - _getattr(self, '__message__'), _getattr(self, '__stacklevel__')) + deprecation(_getattr(self, '__message__'), + stacklevel=_getattr(self, '__stacklevel__')) return isinstance(instance, _getattr(self, '__wrapped__')) def __subclasscheck__(self, instance): - deprecation( - _getattr(self, '__message__'), _getattr(self, '__stacklevel__')) + deprecation(_getattr(self, '__message__'), + stacklevel=_getattr(self, '__stacklevel__')) return issubclass(instance, _getattr(self, '__wrapped__')) def __call__(self, *args, **kwargs): - deprecation( - _getattr(self, '__message__'), _getattr(self, '__stacklevel__')) + deprecation(_getattr(self, '__message__'), + stacklevel=_getattr(self, '__stacklevel__')) return _getattr(self, '__wrapped__')(*args, **kwargs) def __getattribute__(self, name): @@ -95,11 +119,9 @@ class MovedClassProxy(object): type(self).__name__, id(self), wrapped, id(wrapped)) -def _generate_moved_message(kind, old_name, new_name, - message=None, version=None, removal_version=None): - message_components = [ - "%s '%s' has moved to '%s'" % (kind, old_name, new_name), - ] +def _generate_moved_message(prefix, postfix=None, message=None, + version=None, removal_version=None): + message_components = [prefix] if version: message_components.append(" in version '%s'" % version) if removal_version: @@ -109,11 +131,36 @@ def _generate_moved_message(kind, old_name, new_name, else: message_components.append(" and will be removed in version '%s'" % removal_version) + if postfix: + message_components.append(postfix) if message: message_components.append(": %s" % message) return ''.join(message_components) +def renamed_kwarg(old_name, new_name, message=None, + version=None, removal_version=None, stacklevel=3): + """Decorates a kwarg accepting function to deprecate a renamed kwarg.""" + + prefix = _KWARG_MOVED_PREFIX_TPL % old_name + postfix = _KWARG_MOVED_POSTFIX_TPL % new_name + out_message = _generate_moved_message(prefix, postfix=postfix, + message=message, version=version, + removal_version=removal_version) + + def decorator(f): + + @six.wraps(f) + def wrapper(*args, **kwargs): + if old_name in kwargs: + deprecation(out_message, stacklevel=stacklevel) + return f(*args, **kwargs) + + return wrapper + + return decorator + + def _moved_decorator(kind, new_attribute_name, message=None, version=None, removal_version=None): """Decorates a method/property that was moved to another location.""" @@ -134,10 +181,11 @@ def _moved_decorator(kind, new_attribute_name, message=None, else: old_name = ".".join((base_name, old_attribute_name)) new_name = ".".join((base_name, new_attribute_name)) + prefix = _KIND_MOVED_PREFIX_TPL % (kind, old_name, new_name) out_message = _generate_moved_message( - kind, old_name=old_name, new_name=new_name, message=message, + prefix, message=message, version=version, removal_version=removal_version) - deprecation(out_message, 3) + deprecation(out_message, stacklevel=3) return f(self, *args, **kwargs) return wrapper @@ -158,7 +206,8 @@ def moved_class(new_class, old_class_name, old_module_name, message=None, """ old_name = ".".join((old_module_name, old_class_name)) new_name = reflection.get_class_name(new_class) - out_message = _generate_moved_message('Class', old_name, new_name, + prefix = _CLASS_MOVED_PREFIX_TPL % (old_name, new_name) + out_message = _generate_moved_message(prefix, message=message, version=version, removal_version=removal_version) - return MovedClassProxy(new_class, out_message, 3) + return MovedClassProxy(new_class, out_message, stacklevel=3) From 2832d6e677413d08bd1645bb8bd8067b3ab665d6 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 21 Nov 2014 13:07:47 -0800 Subject: [PATCH 106/240] Ensure that the zookeeper backend creates missing atoms When 'create_missing' is true the atom should be created instead of raising an exception; this is used when a flow detail is updated with a new detail and then saved. This also adds test cases that verify this happens so that we verify this on an ongoing basis. Fixes bug 1395812 Change-Id: I4851a08ff1ab4101dbec4a6656177908095c3c52 --- .../persistence/backends/impl_zookeeper.py | 6 ++- taskflow/tests/unit/persistence/base.py | 43 ++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/taskflow/persistence/backends/impl_zookeeper.py b/taskflow/persistence/backends/impl_zookeeper.py index 948c54dc..3cacffec 100644 --- a/taskflow/persistence/backends/impl_zookeeper.py +++ b/taskflow/persistence/backends/impl_zookeeper.py @@ -169,7 +169,11 @@ class ZkConnection(base.Connection): ad_data, _zstat = self._client.get(ad_path) except k_exc.NoNodeError: # Not-existent: create or raise exception. - raise exc.NotFound("No atom details found with id: %s" % ad.uuid) + if not create_missing: + raise exc.NotFound("No atom details found with" + " id: %s" % ad.uuid) + else: + txn.create(ad_path) else: # Existent: read it out. try: diff --git a/taskflow/tests/unit/persistence/base.py b/taskflow/tests/unit/persistence/base.py index 6d96df66..50bb3b3f 100644 --- a/taskflow/tests/unit/persistence/base.py +++ b/taskflow/tests/unit/persistence/base.py @@ -25,7 +25,48 @@ from taskflow.types import failure class PersistenceTestMixin(object): def _get_connection(self): - raise NotImplementedError() + raise NotImplementedError('_get_connection() implementation required') + + def test_task_detail_update_not_existing(self): + lb_id = uuidutils.generate_uuid() + lb_name = 'lb-%s' % (lb_id) + lb = logbook.LogBook(name=lb_name, uuid=lb_id) + fd = logbook.FlowDetail('test', uuid=uuidutils.generate_uuid()) + lb.add(fd) + td = logbook.TaskDetail("detail-1", uuid=uuidutils.generate_uuid()) + fd.add(td) + with contextlib.closing(self._get_connection()) as conn: + conn.save_logbook(lb) + + td2 = logbook.TaskDetail("detail-1", uuid=uuidutils.generate_uuid()) + fd.add(td2) + with contextlib.closing(self._get_connection()) as conn: + conn.update_flow_details(fd) + + with contextlib.closing(self._get_connection()) as conn: + lb2 = conn.get_logbook(lb.uuid) + fd2 = lb2.find(fd.uuid) + self.assertIsNotNone(fd2.find(td.uuid)) + self.assertIsNotNone(fd2.find(td2.uuid)) + + def test_flow_detail_update_not_existing(self): + lb_id = uuidutils.generate_uuid() + lb_name = 'lb-%s' % (lb_id) + lb = logbook.LogBook(name=lb_name, uuid=lb_id) + fd = logbook.FlowDetail('test', uuid=uuidutils.generate_uuid()) + lb.add(fd) + with contextlib.closing(self._get_connection()) as conn: + conn.save_logbook(lb) + + fd2 = logbook.FlowDetail('test-2', uuid=uuidutils.generate_uuid()) + lb.add(fd2) + with contextlib.closing(self._get_connection()) as conn: + conn.save_logbook(lb) + + with contextlib.closing(self._get_connection()) as conn: + lb2 = conn.get_logbook(lb.uuid) + self.assertIsNotNone(lb2.find(fd.uuid)) + self.assertIsNotNone(lb2.find(fd2.uuid)) def test_logbook_save_retrieve(self): lb_id = uuidutils.generate_uuid() From bb8ea5686f683200816ffe9aef8843206b796d51 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 24 Nov 2014 18:10:44 -0800 Subject: [PATCH 107/240] Move scheduler and completer classes to there own modules To allow other components to more easily import these modules without causing circular imports split these two classes into there own modules so that they can be referenced as/where needed in the future. Change-Id: I9cf1fd1dd133be58ef521a8911d1740f1daa9950 --- taskflow/engines/action_engine/completer.py | 114 ++++++++++++ taskflow/engines/action_engine/runtime.py | 190 ++------------------ taskflow/engines/action_engine/scheduler.py | 100 +++++++++++ 3 files changed, 226 insertions(+), 178 deletions(-) create mode 100644 taskflow/engines/action_engine/completer.py create mode 100644 taskflow/engines/action_engine/scheduler.py diff --git a/taskflow/engines/action_engine/completer.py b/taskflow/engines/action_engine/completer.py new file mode 100644 index 00000000..958d5f03 --- /dev/null +++ b/taskflow/engines/action_engine/completer.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from taskflow.engines.action_engine import executor as ex +from taskflow import retry as retry_atom +from taskflow import states as st +from taskflow import task as task_atom +from taskflow.types import failure + + +class Completer(object): + """Completes atoms using actions to complete them.""" + + def __init__(self, runtime): + self._runtime = runtime + self._analyzer = runtime.analyzer + self._retry_action = runtime.retry_action + self._runtime = runtime + self._storage = runtime.storage + self._task_action = runtime.task_action + + def _complete_task(self, task, event, result): + """Completes the given task, processes task failure.""" + if event == ex.EXECUTED: + self._task_action.complete_execution(task, result) + else: + self._task_action.complete_reversion(task, result) + + def resume(self): + """Resumes nodes in the contained graph. + + This is done to allow any previously completed or failed nodes to + be analyzed, there results processed and any potential nodes affected + to be adjusted as needed. + + This should return a set of nodes which should be the initial set of + nodes that were previously not finished (due to a RUNNING or REVERTING + attempt not previously finishing). + """ + for node in self._analyzer.iterate_all_nodes(): + if self._analyzer.get_state(node) == st.FAILURE: + self._process_atom_failure(node, self._storage.get(node.name)) + for retry in self._analyzer.iterate_retries(st.RETRYING): + self._runtime.retry_subflow(retry) + unfinished_nodes = set() + for node in self._analyzer.iterate_all_nodes(): + if self._analyzer.get_state(node) in (st.RUNNING, st.REVERTING): + unfinished_nodes.add(node) + return unfinished_nodes + + def complete(self, node, event, result): + """Performs post-execution completion of a node. + + Returns whether the result should be saved into an accumulator of + failures or whether this should not be done. + """ + if isinstance(node, task_atom.BaseTask): + self._complete_task(node, event, result) + if isinstance(result, failure.Failure): + if event == ex.EXECUTED: + self._process_atom_failure(node, result) + else: + return True + return False + + def _process_atom_failure(self, atom, failure): + """Processes atom failure & applies resolution strategies. + + On atom failure this will find the atoms associated retry controller + and ask that controller for the strategy to perform to resolve that + failure. After getting a resolution strategy decision this method will + then adjust the needed other atoms intentions, and states, ... so that + the failure can be worked around. + """ + retry = self._analyzer.find_atom_retry(atom) + if retry is not None: + # Ask retry controller what to do in case of failure + action = self._retry_action.on_failure(retry, atom, failure) + if action == retry_atom.RETRY: + # Prepare just the surrounding subflow for revert to be later + # retried... + self._storage.set_atom_intention(retry.name, st.RETRY) + self._runtime.reset_subgraph(retry, state=None, + intention=st.REVERT) + elif action == retry_atom.REVERT: + # Ask parent checkpoint. + self._process_atom_failure(retry, failure) + elif action == retry_atom.REVERT_ALL: + # Prepare all flow for revert + self._revert_all() + else: + raise ValueError("Unknown atom failure resolution" + " action '%s'" % action) + else: + # Prepare all flow for revert + self._revert_all() + + def _revert_all(self): + """Attempts to set all nodes to the REVERT intention.""" + self._runtime.reset_nodes(self._analyzer.iterate_all_nodes(), + state=None, intention=st.REVERT) diff --git a/taskflow/engines/action_engine/runtime.py b/taskflow/engines/action_engine/runtime.py index 06959f29..9f02e554 100644 --- a/taskflow/engines/action_engine/runtime.py +++ b/taskflow/engines/action_engine/runtime.py @@ -14,17 +14,16 @@ # License for the specific language governing permissions and limitations # under the License. -from taskflow.engines.action_engine import analyzer as ca -from taskflow.engines.action_engine import executor as ex +from taskflow.engines.action_engine import analyzer as an +from taskflow.engines.action_engine import completer as co from taskflow.engines.action_engine import retry_action as ra from taskflow.engines.action_engine import runner as ru +from taskflow.engines.action_engine import scheduler as sched from taskflow.engines.action_engine import scopes as sc from taskflow.engines.action_engine import task_action as ta -from taskflow import exceptions as excp from taskflow import retry as retry_atom from taskflow import states as st from taskflow import task as task_atom -from taskflow.types import failure from taskflow.utils import misc @@ -52,7 +51,7 @@ class Runtime(object): @misc.cachedproperty def analyzer(self): - return ca.Analyzer(self._compilation, self._storage) + return an.Analyzer(self._compilation, self._storage) @misc.cachedproperty def runner(self): @@ -60,11 +59,11 @@ class Runtime(object): @misc.cachedproperty def completer(self): - return Completer(self) + return co.Completer(self) @misc.cachedproperty def scheduler(self): - return Scheduler(self) + return sched.Scheduler(self) @misc.cachedproperty def retry_action(self): @@ -81,6 +80,9 @@ class Runtime(object): atom, names_only=True)) + # Various helper methods used by the runtime components; not for public + # consumption... + def reset_nodes(self, nodes, state=st.PENDING, intention=st.EXECUTE): for node in nodes: if state: @@ -102,174 +104,6 @@ class Runtime(object): self.reset_nodes(self.analyzer.iterate_subgraph(node), state=state, intention=intention) - -# Various helper methods used by completer and scheduler. -def _retry_subflow(retry, runtime): - runtime.storage.set_atom_intention(retry.name, st.EXECUTE) - runtime.reset_subgraph(retry) - - -class Completer(object): - """Completes atoms using actions to complete them.""" - - def __init__(self, runtime): - self._analyzer = runtime.analyzer - self._retry_action = runtime.retry_action - self._runtime = runtime - self._storage = runtime.storage - self._task_action = runtime.task_action - - def _complete_task(self, task, event, result): - """Completes the given task, processes task failure.""" - if event == ex.EXECUTED: - self._task_action.complete_execution(task, result) - else: - self._task_action.complete_reversion(task, result) - - def resume(self): - """Resumes nodes in the contained graph. - - This is done to allow any previously completed or failed nodes to - be analyzed, there results processed and any potential nodes affected - to be adjusted as needed. - - This should return a set of nodes which should be the initial set of - nodes that were previously not finished (due to a RUNNING or REVERTING - attempt not previously finishing). - """ - for node in self._analyzer.iterate_all_nodes(): - if self._analyzer.get_state(node) == st.FAILURE: - self._process_atom_failure(node, self._storage.get(node.name)) - for retry in self._analyzer.iterate_retries(st.RETRYING): - _retry_subflow(retry, self._runtime) - unfinished_nodes = set() - for node in self._analyzer.iterate_all_nodes(): - if self._analyzer.get_state(node) in (st.RUNNING, st.REVERTING): - unfinished_nodes.add(node) - return unfinished_nodes - - def complete(self, node, event, result): - """Performs post-execution completion of a node. - - Returns whether the result should be saved into an accumulator of - failures or whether this should not be done. - """ - if isinstance(node, task_atom.BaseTask): - self._complete_task(node, event, result) - if isinstance(result, failure.Failure): - if event == ex.EXECUTED: - self._process_atom_failure(node, result) - else: - return True - return False - - def _process_atom_failure(self, atom, failure): - """Processes atom failure & applies resolution strategies. - - On atom failure this will find the atoms associated retry controller - and ask that controller for the strategy to perform to resolve that - failure. After getting a resolution strategy decision this method will - then adjust the needed other atoms intentions, and states, ... so that - the failure can be worked around. - """ - retry = self._analyzer.find_atom_retry(atom) - if retry: - # Ask retry controller what to do in case of failure - action = self._retry_action.on_failure(retry, atom, failure) - if action == retry_atom.RETRY: - # Prepare subflow for revert - self._storage.set_atom_intention(retry.name, st.RETRY) - self._runtime.reset_subgraph(retry, state=None, - intention=st.REVERT) - elif action == retry_atom.REVERT: - # Ask parent checkpoint - self._process_atom_failure(retry, failure) - elif action == retry_atom.REVERT_ALL: - # Prepare all flow for revert - self._revert_all() - else: - # Prepare all flow for revert - self._revert_all() - - def _revert_all(self): - """Attempts to set all nodes to the REVERT intention.""" - self._runtime.reset_nodes(self._analyzer.iterate_all_nodes(), - state=None, intention=st.REVERT) - - -class Scheduler(object): - """Schedules atoms using actions to schedule.""" - - def __init__(self, runtime): - self._analyzer = runtime.analyzer - self._retry_action = runtime.retry_action - self._runtime = runtime - self._storage = runtime.storage - self._task_action = runtime.task_action - - def _schedule_node(self, node): - """Schedule a single node for execution.""" - # TODO(harlowja): we need to rework this so that we aren't doing type - # checking here, type checking usually means something isn't done right - # and usually will limit extensibility in the future. - if isinstance(node, task_atom.BaseTask): - return self._schedule_task(node) - elif isinstance(node, retry_atom.Retry): - return self._schedule_retry(node) - else: - raise TypeError("Unknown how to schedule atom '%s' (%s)" - % (node, type(node))) - - def _schedule_retry(self, retry): - """Schedules the given retry atom for *future* completion. - - Depending on the atoms stored intention this may schedule the retry - atom for reversion or execution. - """ - intention = self._storage.get_atom_intention(retry.name) - if intention == st.EXECUTE: - return self._retry_action.execute(retry) - elif intention == st.REVERT: - return self._retry_action.revert(retry) - elif intention == st.RETRY: - self._retry_action.change_state(retry, st.RETRYING) - _retry_subflow(retry, self._runtime) - return self._retry_action.execute(retry) - else: - raise excp.ExecutionFailure("Unknown how to schedule retry with" - " intention: %s" % intention) - - def _schedule_task(self, task): - """Schedules the given task atom for *future* completion. - - Depending on the atoms stored intention this may schedule the task - atom for reversion or execution. - """ - intention = self._storage.get_atom_intention(task.name) - if intention == st.EXECUTE: - return self._task_action.schedule_execution(task) - elif intention == st.REVERT: - return self._task_action.schedule_reversion(task) - else: - raise excp.ExecutionFailure("Unknown how to schedule task with" - " intention: %s" % intention) - - def schedule(self, nodes): - """Schedules the provided nodes for *future* completion. - - This method should schedule a future for each node provided and return - a set of those futures to be waited on (or used for other similar - purposes). It should also return any failure objects that represented - scheduling failures that may have occurred during this scheduling - process. - """ - futures = set() - for node in nodes: - try: - futures.add(self._schedule_node(node)) - except Exception: - # Immediately stop scheduling future work so that we can - # exit execution early (rather than later) if a single task - # fails to schedule correctly. - return (futures, [failure.Failure()]) - return (futures, []) + def retry_subflow(self, retry): + self.storage.set_atom_intention(retry.name, st.EXECUTE) + self.reset_subgraph(retry) diff --git a/taskflow/engines/action_engine/scheduler.py b/taskflow/engines/action_engine/scheduler.py new file mode 100644 index 00000000..266c9ebb --- /dev/null +++ b/taskflow/engines/action_engine/scheduler.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from taskflow import exceptions as excp +from taskflow import retry as retry_atom +from taskflow import states as st +from taskflow import task as task_atom +from taskflow.types import failure + + +class Scheduler(object): + """Schedules atoms using actions to schedule.""" + + def __init__(self, runtime): + self._runtime = runtime + self._analyzer = runtime.analyzer + self._retry_action = runtime.retry_action + self._runtime = runtime + self._storage = runtime.storage + self._task_action = runtime.task_action + + def _schedule_node(self, node): + """Schedule a single node for execution.""" + # TODO(harlowja): we need to rework this so that we aren't doing type + # checking here, type checking usually means something isn't done right + # and usually will limit extensibility in the future. + if isinstance(node, task_atom.BaseTask): + return self._schedule_task(node) + elif isinstance(node, retry_atom.Retry): + return self._schedule_retry(node) + else: + raise TypeError("Unknown how to schedule atom '%s' (%s)" + % (node, type(node))) + + def _schedule_retry(self, retry): + """Schedules the given retry atom for *future* completion. + + Depending on the atoms stored intention this may schedule the retry + atom for reversion or execution. + """ + intention = self._storage.get_atom_intention(retry.name) + if intention == st.EXECUTE: + return self._retry_action.execute(retry) + elif intention == st.REVERT: + return self._retry_action.revert(retry) + elif intention == st.RETRY: + self._retry_action.change_state(retry, st.RETRYING) + self._runtime.retry_subflow(retry) + return self._retry_action.execute(retry) + else: + raise excp.ExecutionFailure("Unknown how to schedule retry with" + " intention: %s" % intention) + + def _schedule_task(self, task): + """Schedules the given task atom for *future* completion. + + Depending on the atoms stored intention this may schedule the task + atom for reversion or execution. + """ + intention = self._storage.get_atom_intention(task.name) + if intention == st.EXECUTE: + return self._task_action.schedule_execution(task) + elif intention == st.REVERT: + return self._task_action.schedule_reversion(task) + else: + raise excp.ExecutionFailure("Unknown how to schedule task with" + " intention: %s" % intention) + + def schedule(self, nodes): + """Schedules the provided nodes for *future* completion. + + This method should schedule a future for each node provided and return + a set of those futures to be waited on (or used for other similar + purposes). It should also return any failure objects that represented + scheduling failures that may have occurred during this scheduling + process. + """ + futures = set() + for node in nodes: + try: + futures.add(self._schedule_node(node)) + except Exception: + # Immediately stop scheduling future work so that we can + # exit execution early (rather than later) if a single task + # fails to schedule correctly. + return (futures, [failure.Failure()]) + return (futures, []) From cf85dd0f61dcf402c42b3649b9ed449142a1ac56 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 24 Nov 2014 17:07:52 -0800 Subject: [PATCH 108/240] Remove default setting of 'mysql_traditional_mode' The sqlalchemy versions we are using/supporting (0.6+) have the detection of the mysql mode built-in so only activate the connect setting if we are somehow overriden by a user who knows what they are doing. Fixes bug 1396278 Change-Id: If2226d3e9f921a1c5f62a6727016fe86cd50a9b5 --- taskflow/persistence/backends/impl_sqlalchemy.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/taskflow/persistence/backends/impl_sqlalchemy.py b/taskflow/persistence/backends/impl_sqlalchemy.py index 4b12b782..2ab36662 100644 --- a/taskflow/persistence/backends/impl_sqlalchemy.py +++ b/taskflow/persistence/backends/impl_sqlalchemy.py @@ -257,8 +257,6 @@ class SQLAlchemyBackend(base.Backend): if _as_bool(conf.pop('checkout_ping', True)): sa.event.listen(engine, 'checkout', _ping_listener) mode = None - if _as_bool(conf.pop('mysql_traditional_mode', True)): - mode = 'TRADITIONAL' if 'mysql_sql_mode' in conf: mode = conf.pop('mysql_sql_mode') if mode is not None: From 265181f573b025b4038774b96e5a40e6bb2f1672 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 18 Nov 2014 18:07:37 -0800 Subject: [PATCH 109/240] Use a metaclass to dynamically add testcases to example runner Instead of using a custom update() mechanism just take advantage of a metaclass that can dynamically add our desired test functions on in a more pythonic manner. Change-Id: I8f4940f85dd7b5255c181795606ac76ca5605baa --- taskflow/tests/test_examples.py | 75 +++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 32 deletions(-) diff --git a/taskflow/tests/test_examples.py b/taskflow/tests/test_examples.py index 2631cd47..a7a297c3 100644 --- a/taskflow/tests/test_examples.py +++ b/taskflow/tests/test_examples.py @@ -28,22 +28,37 @@ examples is indeterministic (due to hash randomization for example). """ +import keyword import os import re import subprocess import sys -import taskflow.test +import six + +from taskflow import test ROOT_DIR = os.path.abspath( os.path.dirname( os.path.dirname( os.path.dirname(__file__)))) +# This is used so that any uuid like data being output is removed (since it +# will change per test run and will invalidate the deterministic output that +# we expect to be able to check). UUID_RE = re.compile('XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX' .replace('X', '[0-9a-f]')) +def safe_filename(filename): + # Translates a filename into a method name, returns falsey if not + # possible to perform this translation... + name = re.sub("[^a-zA-Z0-9_]+", "_", filename) + if not name or re.match(r"^[_]+$", name) or keyword.iskeyword(name): + return False + return name + + def root_path(*args): return os.path.join(ROOT_DIR, *args) @@ -71,7 +86,7 @@ def expected_output_path(name): return root_path('taskflow', 'examples', '%s.out.txt' % name) -def list_examples(): +def iter_examples(): examples_dir = root_path('taskflow', 'examples') for filename in os.listdir(examples_dir): path = os.path.join(examples_dir, filename) @@ -80,38 +95,34 @@ def list_examples(): name, ext = os.path.splitext(filename) if ext != ".py": continue - bad_endings = [] - for i in ("utils", "no_test"): - if name.endswith(i): - bad_endings.append(True) - if not any(bad_endings): - yield name + if not any(name.endswith(i) for i in ("utils", "no_test")): + safe_name = safe_filename(name) + if safe_name: + yield name, safe_name -class ExamplesTestCase(taskflow.test.TestCase): - @classmethod - def update(cls): - """For each example, adds on a test method. +class ExampleAdderMeta(type): + """Translates examples into test cases/methods.""" - This newly created test method will then be activated by the testing - framework when it scans for and runs tests. This makes for a elegant - and simple way to ensure that all of the provided examples - actually work. - """ - def add_test_method(name, method_name): + def __new__(cls, name, parents, dct): + + def generate_test(example_name): def test_example(self): - self._check_example(name) - test_example.__name__ = method_name - setattr(cls, method_name, test_example) + self._check_example(example_name) + return test_example - for name in list_examples(): - safe_name = str(re.sub("[^a-zA-Z0-9_]+", "_", name)) - if re.match(r"^[_]+$", safe_name): - continue - add_test_method(name, 'test_%s' % safe_name) + for example_name, safe_name in iter_examples(): + test_name = 'test_%s' % safe_name + dct[test_name] = generate_test(example_name) + + return type.__new__(cls, name, parents, dct) + + +@six.add_metaclass(ExampleAdderMeta) +class ExamplesTestCase(test.TestCase): + """Runs the examples, and checks the outputs against expected outputs.""" def _check_example(self, name): - """Runs the example, and checks the output against expected output.""" output = run_example(name) eop = expected_output_path(name) if os.path.isfile(eop): @@ -123,14 +134,14 @@ class ExamplesTestCase(taskflow.test.TestCase): expected_output = UUID_RE.sub('', expected_output) self.assertEqual(output, expected_output) -ExamplesTestCase.update() - def make_output_files(): """Generate output files for all examples.""" - for name in list_examples(): - output = run_example(name) - with open(expected_output_path(name), 'w') as f: + for example_name, _safe_name in iter_examples(): + print("Running %s" % example_name) + print("Please wait...") + output = run_example(example_name) + with open(expected_output_path(example_name), 'w') as f: f.write(output) From 95e94f768a49e4fa7317e86c288450efd3aa922c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 27 Nov 2014 21:56:07 -0800 Subject: [PATCH 110/240] Include documentation of the utility modules Add a documentation section that uses the automodule sphinx documentation generator to build a utility page that shows people what the utility modules taskflow uses for *internal* usage are and include a warning to try to avoid end-users from using these modules (better to warn than not warn). This also adds on docstrings to various functions that were missing it so that they appear correctly in the generated documentation. Change-Id: Ibd5695927601614f31793ea5450887694f4320ae --- doc/source/index.rst | 11 +++----- doc/source/utils.rst | 49 +++++++++++++++++++++++++++++++++++ taskflow/utils/deprecation.py | 9 ++++--- taskflow/utils/kazoo_utils.py | 8 ++++-- taskflow/utils/misc.py | 26 +++++++++++++++---- 5 files changed, 86 insertions(+), 17 deletions(-) create mode 100644 doc/source/utils.rst diff --git a/doc/source/index.rst b/doc/source/index.rst index d3820470..657a08be 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -70,13 +70,9 @@ TaskFlow into your project: ``[TaskFlow]`` to your emails subject to get an even faster response). * Follow (or at least attempt to follow) some of the established `best practices`_ (feel free to add your own suggested best practices). - -.. warning:: - - External usage of internal helpers and other internal utility functions - and modules should be kept to a *minimum* as these may be altered, - refactored or moved *without* notice. If you are unsure whether to use - a function, class, or module, please ask (see above). +* Keep in touch with the team (see above); we are all friendly and enjoy + knowing your use cases and learning how we can help make your lives easier + by adding or adjusting functionality in this library. .. _IRC: irc://chat.freenode.net/openstack-state-management .. _best practices: http://wiki.openstack.org/wiki/TaskFlow/Best_practices @@ -92,6 +88,7 @@ Miscellaneous exceptions states types + utils Indices and tables ================== diff --git a/doc/source/utils.rst b/doc/source/utils.rst new file mode 100644 index 00000000..968c2c04 --- /dev/null +++ b/doc/source/utils.rst @@ -0,0 +1,49 @@ +--------- +Utilities +--------- + +.. warning:: + + External usage of internal utility functions and modules should be kept + to a **minimum** as they may be altered, refactored or moved to other + locations **without** notice (and without the typical deprecation cycle). + +Async +~~~~~ + +.. automodule:: taskflow.utils.async_utils + +Deprecation +~~~~~~~~~~~ + +.. automodule:: taskflow.utils.deprecation + +Kazoo +~~~~~ + +.. automodule:: taskflow.utils.kazoo_utils + +Locks +~~~~~ + +.. automodule:: taskflow.utils.lock_utils + +Miscellaneous +~~~~~~~~~~~~~ + +.. automodule:: taskflow.utils.misc + +Persistence +~~~~~~~~~~~ + +.. automodule:: taskflow.utils.persistence_utils + +Reflection +~~~~~~~~~~ + +.. automodule:: taskflow.utils.reflection + +Threading +~~~~~~~~~ + +.. automodule:: taskflow.utils.threading_utils diff --git a/taskflow/utils/deprecation.py b/taskflow/utils/deprecation.py index a5c7148f..db9c3790 100644 --- a/taskflow/utils/deprecation.py +++ b/taskflow/utils/deprecation.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -import functools import warnings import six @@ -193,8 +192,12 @@ def _moved_decorator(kind, new_attribute_name, message=None, return decorator -"""Decorates a *instance* property that was moved to another location.""" -moved_property = functools.partial(_moved_decorator, 'Property') +def moved_property(new_attribute_name, message=None, + version=None, removal_version=None): + """Decorates a *instance* property that was moved to another location.""" + + return _moved_decorator('Property', new_attribute_name, message=message, + version=version, removal_version=removal_version) def moved_class(new_class, old_class_name, old_module_name, message=None, diff --git a/taskflow/utils/kazoo_utils.py b/taskflow/utils/kazoo_utils.py index 93da2cdd..0a9922bb 100644 --- a/taskflow/utils/kazoo_utils.py +++ b/taskflow/utils/kazoo_utils.py @@ -95,8 +95,12 @@ class KazooTransactionException(k_exc.KazooException): def checked_commit(txn): - # Until https://github.com/python-zk/kazoo/pull/224 is fixed we have - # to workaround the transaction failing silently. + """Commits a kazoo transcation and validates the result. + + NOTE(harlowja): Until https://github.com/python-zk/kazoo/pull/224 is fixed + or a similar pull request is merged we have to workaround the transaction + failing silently. + """ if not txn.operations: return [] results = txn.commit() diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index b4357cf4..583cdd1a 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -342,11 +342,27 @@ def capture_failure(): For example:: - except Exception: - with capture_failure() as fail: - LOG.warn("Activating cleanup") - cleanup() - save_failure(fail) + >>> from taskflow.utils import misc + >>> + >>> def cleanup(): + ... pass + ... + >>> + >>> def save_failure(f): + ... print("Saving %s" % f) + ... + >>> + >>> try: + ... raise IOError("Broken") + ... except Exception: + ... with misc.capture_failure() as fail: + ... print("Activating cleanup") + ... cleanup() + ... save_failure(fail) + ... + Activating cleanup + Saving Failure: IOError: Broken + """ exc_info = sys.exc_info() if not any(exc_info): From e07fb21a1dda9e6c6698e4235064d405b06dec9d Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 28 Nov 2014 15:03:34 -0800 Subject: [PATCH 111/240] Avoid deepcopying exception values Since failure objects are by design meant to be immutable we don't/shouldn't need to deepcopy the exception value; so instead just do a shallow copy and depend on the semantics that failure objects are immutable to avoid any subsequent issues here. Change-Id: Id1f9ae04b330ab8c16ab2f7d1e877032639f1cb3 --- taskflow/types/failure.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/taskflow/types/failure.py b/taskflow/types/failure.py index c905277c..0f7f3d2c 100644 --- a/taskflow/types/failure.py +++ b/taskflow/types/failure.py @@ -25,13 +25,13 @@ from taskflow.utils import reflection def _copy_exc_info(exc_info): - """Make copy of exception info tuple, as deep as possible.""" if exc_info is None: return None exc_type, exc_value, tb = exc_info - # NOTE(imelnikov): there is no need to copy type, and - # we can't copy traceback. - return (exc_type, copy.deepcopy(exc_value), tb) + # NOTE(imelnikov): there is no need to copy the exception type, and + # a shallow copy of the value is fine and we can't copy the traceback since + # it contains reference to the internal stack frames... + return (exc_type, copy.copy(exc_value), tb) def _are_equal_exc_info_tuples(ei1, ei2): From 96759647dd3ccc2a362154226d1fff3f93fe9659 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 28 Nov 2014 15:09:33 -0800 Subject: [PATCH 112/240] Add link to issue 17911 The ongoing development in upstream python may produce a useful class that we can use in the future to reduce this objects code; so this adds a link to that for future reference. Change-Id: I86baa096e113cccbb4915ad4fd6fb8cc0313e0a0 --- taskflow/types/failure.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/taskflow/types/failure.py b/taskflow/types/failure.py index c905277c..303118a8 100644 --- a/taskflow/types/failure.py +++ b/taskflow/types/failure.py @@ -107,6 +107,10 @@ class Failure(object): python has where foreign modules can be imported, causing those modules to have code ran when this happens, and this can cause issues and side-effects that the receiver would not have intended to have caused). + + TODO(harlowja): when/if http://bugs.python.org/issue17911 merges and + becomes available for use we should be able to use that and simplify the + methods and contents of this object. """ DICT_VERSION = 1 From 178f2799c47e1a400fcf1c8b138c7060649463a8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 21 Nov 2014 14:31:08 -0800 Subject: [PATCH 113/240] Move the _pformat() method to be a classmethod To avoid creating nested functions when a class one will suffice just switch to using a class method for the nested _pformat() function instead. This also makes sure that the cause of a exception is not none before attempting to format it (and its children) and also passes along a lines list that is appended/extended instead of each function call creating its own version which is later merged. Change-Id: Ie84b004c115e488c87f9829bfcf4d8f66ebb660f --- taskflow/exceptions.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/taskflow/exceptions.py b/taskflow/exceptions.py index 876a3e3b..1c6bc9f1 100644 --- a/taskflow/exceptions.py +++ b/taskflow/exceptions.py @@ -46,21 +46,26 @@ class TaskFlowException(Exception): """Pretty formats a taskflow exception + any connected causes.""" if indent < 0: raise ValueError("indent must be greater than or equal to zero") + return "\n".join(self._pformat(self, [], 0, + indent=indent, indent_text=indent_text)) - def _format(excp, indent_by): - lines = [] - for line in traceback.format_exception_only(type(excp), excp): - # We'll add our own newlines on at the end of formatting. - if line.endswith("\n"): - line = line[0:-1] - lines.append((indent_text * indent_by) + line) - try: - lines.extend(_format(excp.cause, indent_by + indent)) - except AttributeError: - pass - return lines - - return "\n".join(_format(self, 0)) + @classmethod + def _pformat(cls, excp, lines, current_indent, indent=2, indent_text=" "): + line_prefix = indent_text * current_indent + for line in traceback.format_exception_only(type(excp), excp): + # We'll add our own newlines on at the end of formatting. + if line.endswith("\n"): + line = line[0:-1] + lines.append(line_prefix + line) + try: + cause = excp.cause + except AttributeError: + pass + else: + if cause is not None: + cls._pformat(cause, lines, current_indent + indent, + indent=indent, indent_text=indent_text) + return lines # Errors related to storage or operations on storage units. From 35fcd9060fc51eeaf52177398a8c4e26857b038d Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 1 Dec 2014 18:13:26 -0800 Subject: [PATCH 114/240] Use a module level constant to provide the DEFAULT_LISTEN_FOR Instead of repeating the same tuple over and over again just provide a module level constant in the listener base module that defines the default listen_for tuple that is then used in all subclasses as the default (if those subclasses support forwarding the listen_for states in the first place). This makes it so that users of listeners can more easily figure out the default to pass, without having to get involved in the inner details. Change-Id: I3041db4d7eabfbe8d0b2cc267cfca0e46c0b7139 --- taskflow/listeners/base.py | 7 +++++-- taskflow/listeners/logging.py | 9 ++++----- taskflow/listeners/printing.py | 5 ++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/taskflow/listeners/base.py b/taskflow/listeners/base.py index e1d475f7..739db49c 100644 --- a/taskflow/listeners/base.py +++ b/taskflow/listeners/base.py @@ -32,6 +32,9 @@ LOG = logging.getLogger(__name__) # do not produce results. FINISH_STATES = (states.FAILURE, states.SUCCESS) +# What is listened for by default... +DEFAULT_LISTEN_FOR = (notifier.Notifier.ANY,) + class ListenerBase(object): """Base class for listeners. @@ -47,8 +50,8 @@ class ListenerBase(object): """ def __init__(self, engine, - task_listen_for=(notifier.Notifier.ANY,), - flow_listen_for=(notifier.Notifier.ANY,)): + task_listen_for=DEFAULT_LISTEN_FOR, + flow_listen_for=DEFAULT_LISTEN_FOR): if not task_listen_for: task_listen_for = [] if not flow_listen_for: diff --git a/taskflow/listeners/logging.py b/taskflow/listeners/logging.py index 3629bb2c..e1459417 100644 --- a/taskflow/listeners/logging.py +++ b/taskflow/listeners/logging.py @@ -22,7 +22,6 @@ import sys from taskflow.listeners import base from taskflow import states from taskflow.types import failure -from taskflow.types import notifier LOG = logging.getLogger(__name__) @@ -52,8 +51,8 @@ class LoggingListener(base.LoggingBase): is provided. """ def __init__(self, engine, - task_listen_for=(notifier.Notifier.ANY,), - flow_listen_for=(notifier.Notifier.ANY,), + task_listen_for=base.DEFAULT_LISTEN_FOR, + flow_listen_for=base.DEFAULT_LISTEN_FOR, log=None, level=logging.DEBUG): super(LoggingListener, self).__init__(engine, @@ -100,8 +99,8 @@ class DynamicLoggingListener(base.ListenerBase): """ def __init__(self, engine, - task_listen_for=(notifier.Notifier.ANY,), - flow_listen_for=(notifier.Notifier.ANY,), + task_listen_for=base.DEFAULT_LISTEN_FOR, + flow_listen_for=base.DEFAULT_LISTEN_FOR, log=None, failure_level=logging.WARNING, level=logging.DEBUG): super(DynamicLoggingListener, self).__init__( diff --git a/taskflow/listeners/printing.py b/taskflow/listeners/printing.py index a7a137b1..397914ed 100644 --- a/taskflow/listeners/printing.py +++ b/taskflow/listeners/printing.py @@ -20,14 +20,13 @@ import sys import traceback from taskflow.listeners import base -from taskflow.types import notifier class PrintingListener(base.LoggingBase): """Writes the task and flow notifications messages to stdout or stderr.""" def __init__(self, engine, - task_listen_for=(notifier.Notifier.ANY,), - flow_listen_for=(notifier.Notifier.ANY,), + task_listen_for=base.DEFAULT_LISTEN_FOR, + flow_listen_for=base.DEFAULT_LISTEN_FOR, stderr=False): super(PrintingListener, self).__init__(engine, task_listen_for=task_listen_for, From 1e8fabd0cbeb88d267b3068d3f92ceb8333a31dc Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 1 Dec 2014 19:22:34 -0800 Subject: [PATCH 115/240] Split the scheduler into sub-schedulers Instead of having a larger scheduler class that contains logic for both the retry routine and the task routine split this into two classes and have the scheduler class use those sub-schedulers for internal scheduling. Change-Id: I6309a5fd172d5b20a01a2ba8b3e4cf8512d085fb --- taskflow/engines/action_engine/scheduler.py | 54 +++++++++++++-------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/taskflow/engines/action_engine/scheduler.py b/taskflow/engines/action_engine/scheduler.py index 266c9ebb..82bacbe4 100644 --- a/taskflow/engines/action_engine/scheduler.py +++ b/taskflow/engines/action_engine/scheduler.py @@ -21,31 +21,17 @@ from taskflow import task as task_atom from taskflow.types import failure -class Scheduler(object): - """Schedules atoms using actions to schedule.""" - +class _RetryScheduler(object): def __init__(self, runtime): self._runtime = runtime - self._analyzer = runtime.analyzer self._retry_action = runtime.retry_action - self._runtime = runtime self._storage = runtime.storage - self._task_action = runtime.task_action - def _schedule_node(self, node): - """Schedule a single node for execution.""" - # TODO(harlowja): we need to rework this so that we aren't doing type - # checking here, type checking usually means something isn't done right - # and usually will limit extensibility in the future. - if isinstance(node, task_atom.BaseTask): - return self._schedule_task(node) - elif isinstance(node, retry_atom.Retry): - return self._schedule_retry(node) - else: - raise TypeError("Unknown how to schedule atom '%s' (%s)" - % (node, type(node))) + @staticmethod + def handles(atom): + return isinstance(atom, retry_atom.Retry) - def _schedule_retry(self, retry): + def schedule(self, retry): """Schedules the given retry atom for *future* completion. Depending on the atoms stored intention this may schedule the retry @@ -64,7 +50,17 @@ class Scheduler(object): raise excp.ExecutionFailure("Unknown how to schedule retry with" " intention: %s" % intention) - def _schedule_task(self, task): + +class _TaskScheduler(object): + def __init__(self, runtime): + self._storage = runtime.storage + self._task_action = runtime.task_action + + @staticmethod + def handles(atom): + return isinstance(atom, task_atom.BaseTask) + + def schedule(self, task): """Schedules the given task atom for *future* completion. Depending on the atoms stored intention this may schedule the task @@ -79,6 +75,24 @@ class Scheduler(object): raise excp.ExecutionFailure("Unknown how to schedule task with" " intention: %s" % intention) + +class Scheduler(object): + """Schedules atoms using actions to schedule.""" + + def __init__(self, runtime): + self._schedulers = [ + _RetryScheduler(runtime), + _TaskScheduler(runtime), + ] + + def _schedule_node(self, node): + """Schedule a single node for execution.""" + for sched in self._schedulers: + if sched.handles(node): + return sched.schedule(node) + else: + raise TypeError("Unknown how to schedule '%s'" % node) + def schedule(self, nodes): """Schedules the provided nodes for *future* completion. From 1de8bbd8382ecda419e9d7753c2be5dabdbfcbd6 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 24 Nov 2014 11:50:28 -0800 Subject: [PATCH 116/240] Add a claims listener that connects job claims to engines To make it easily possible to stop running a engine that was created from a job, add a claims listener that will be called on state changes that an engine progresses through. During those state changes the jobboard will be queried to determine if the job is still claimed by the respective owner; if not the engine will be suspended and further work will stop. Change-Id: I8bbc6a3e03746ba0a7c74139cf9e230631d80d8f --- doc/source/notifications.rst | 5 + taskflow/listeners/claims.py | 100 +++++++++++++++++++ taskflow/tests/unit/test_listeners.py | 138 ++++++++++++++++++++++++++ 3 files changed, 243 insertions(+) create mode 100644 taskflow/listeners/claims.py diff --git a/doc/source/notifications.rst b/doc/source/notifications.rst index aaa5c064..3b857506 100644 --- a/doc/source/notifications.rst +++ b/doc/source/notifications.rst @@ -168,3 +168,8 @@ Timing listener .. autoclass:: taskflow.listeners.timing.TimingListener .. autoclass:: taskflow.listeners.timing.PrintingTimingListener + +Claim listener +-------------- + +.. autoclass:: taskflow.listeners.claims.CheckingClaimListener diff --git a/taskflow/listeners/claims.py b/taskflow/listeners/claims.py new file mode 100644 index 00000000..3fbc15d1 --- /dev/null +++ b/taskflow/listeners/claims.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import + +import logging + +import six + +from taskflow import exceptions +from taskflow.listeners import base +from taskflow import states + +LOG = logging.getLogger(__name__) + + +class CheckingClaimListener(base.ListenerBase): + """Listener that interacts [engine, job, jobboard]; ensures claim is valid. + + This listener (or a derivative) can be associated with an engines + notification system after the job has been claimed (so that the jobs work + can be worked on by that engine). This listener (after associated) will + check that the job is still claimed *whenever* the engine notifies of a + task or flow state change. If the job is not claimed when a state change + occurs, a associated handler (or the default) will be activated to + determine how to react to this *hopefully* exceptional case. + + NOTE(harlowja): this may create more traffic than desired to the + jobboard backend (zookeeper or other), since the amount of state change + per task and flow is non-zero (and checking during each state change will + result in quite a few calls to that management system to check the jobs + claim status); this could be later optimized to check less (or only check + on a smaller set of states) + + NOTE(harlowja): if a custom ``on_job_loss`` callback is provided it must + accept three positional arguments, the first being the current engine being + ran, the second being the 'task/flow' state and the third being the details + that were sent from the engine to listeners for inspection. + """ + + def __init__(self, engine, job, board, owner, on_job_loss=None): + super(CheckingClaimListener, self).__init__(engine) + self._job = job + self._board = board + self._owner = owner + if on_job_loss is None: + self._on_job_loss = self._suspend_engine_on_loss + else: + if not six.callable(on_job_loss): + raise ValueError("Custom 'on_job_loss' handler must be" + " callable") + self._on_job_loss = on_job_loss + + def _suspend_engine_on_loss(self, engine, state, details): + """The default strategy for handling claims being lost.""" + try: + engine.suspend() + except exceptions.TaskFlowException as e: + LOG.warn("Failed suspending engine '%s', (previously owned by" + " '%s'):\n%s", engine, self._owner, e.pformat()) + + def _flow_receiver(self, state, details): + self._claim_checker(state, details) + + def _task_receiver(self, state, details): + self._claim_checker(state, details) + + def _has_been_lost(self): + try: + job_state = self._job.state + job_owner = self._board.find_owner(self._job) + except (exceptions.NotFound, exceptions.JobFailure): + return True + else: + if job_state == states.UNCLAIMED or self._owner != job_owner: + return True + else: + return False + + def _claim_checker(self, state, details): + if not self._has_been_lost(): + LOG.debug("Job '%s' is still claimed (actively owned by '%s')", + self._job, self._owner) + else: + LOG.warn("Job '%s' has lost its claim (previously owned by '%s')", + self._job, self._owner) + self._on_job_loss(self._engine, state, details) diff --git a/taskflow/tests/unit/test_listeners.py b/taskflow/tests/unit/test_listeners.py index 6ba97d6d..210fe798 100644 --- a/taskflow/tests/unit/test_listeners.py +++ b/taskflow/tests/unit/test_listeners.py @@ -18,18 +18,27 @@ import contextlib import logging import time +from oslo.serialization import jsonutils +import six +from zake import fake_client + import taskflow.engines from taskflow import exceptions as exc +from taskflow.jobs import backends as jobs +from taskflow.listeners import claims from taskflow.listeners import logging as logging_listeners from taskflow.listeners import timing from taskflow.patterns import linear_flow as lf from taskflow.persistence.backends import impl_memory +from taskflow import states from taskflow import task from taskflow import test from taskflow.test import mock from taskflow.tests import utils as test_utils +from taskflow.utils import misc from taskflow.utils import persistence_utils from taskflow.utils import reflection +from taskflow.utils import threading_utils _LOG_LEVELS = frozenset([ @@ -64,6 +73,135 @@ class EngineMakerMixin(object): return e +class TestClaimListener(test.TestCase, EngineMakerMixin): + def _make_dummy_flow(self, count): + f = lf.Flow('root') + for i in range(0, count): + f.add(test_utils.ProvidesRequiresTask('%s_test' % i, [], [])) + return f + + def setUp(self): + super(TestClaimListener, self).setUp() + self.client = fake_client.FakeClient() + self.addCleanup(self.client.stop) + self.board = jobs.fetch('test', 'zookeeper', client=self.client) + self.addCleanup(self.board.close) + self.board.connect() + + def _post_claim_job(self, job_name, book=None, details=None): + arrived = threading_utils.Event() + + def set_on_children(children): + if children: + arrived.set() + + self.client.ChildrenWatch("/taskflow", set_on_children) + job = self.board.post('test-1') + + # Make sure it arrived and claimed before doing further work... + self.assertTrue(arrived.wait(test_utils.WAIT_TIMEOUT)) + arrived.clear() + self.board.claim(job, self.board.name) + self.assertTrue(arrived.wait(test_utils.WAIT_TIMEOUT)) + self.assertEqual(states.CLAIMED, job.state) + + return job + + def _destroy_locks(self): + children = self.client.storage.get_children("/taskflow", + only_direct=False) + removed = 0 + for p, data in six.iteritems(children): + if p.endswith(".lock"): + self.client.storage.pop(p) + removed += 1 + return removed + + def _change_owner(self, new_owner): + children = self.client.storage.get_children("/taskflow", + only_direct=False) + altered = 0 + for p, data in six.iteritems(children): + if p.endswith(".lock"): + self.client.set(p, misc.binary_encode( + jsonutils.dumps({'owner': new_owner}))) + altered += 1 + return altered + + def test_bad_create(self): + job = self._post_claim_job('test') + f = self._make_dummy_flow(10) + e = self._make_engine(f) + self.assertRaises(ValueError, claims.CheckingClaimListener, + e, job, self.board, self.board.name, + on_job_loss=1) + + def test_claim_lost_suspended(self): + job = self._post_claim_job('test') + f = self._make_dummy_flow(10) + e = self._make_engine(f) + + try_destroy = True + ran_states = [] + with claims.CheckingClaimListener(e, job, + self.board, self.board.name): + for state in e.run_iter(): + ran_states.append(state) + if state == states.SCHEDULING and try_destroy: + try_destroy = bool(self._destroy_locks()) + + self.assertEqual(states.SUSPENDED, e.storage.get_flow_state()) + self.assertEqual(1, ran_states.count(states.ANALYZING)) + self.assertEqual(1, ran_states.count(states.SCHEDULING)) + self.assertEqual(1, ran_states.count(states.WAITING)) + + def test_claim_lost_custom_handler(self): + job = self._post_claim_job('test') + f = self._make_dummy_flow(10) + e = self._make_engine(f) + + handler = mock.MagicMock() + ran_states = [] + try_destroy = True + destroyed_at = -1 + with claims.CheckingClaimListener(e, job, self.board, + self.board.name, + on_job_loss=handler): + for i, state in enumerate(e.run_iter()): + ran_states.append(state) + if state == states.SCHEDULING and try_destroy: + destroyed = bool(self._destroy_locks()) + if destroyed: + destroyed_at = i + try_destroy = False + + self.assertTrue(handler.called) + self.assertEqual(10, ran_states.count(states.SCHEDULING)) + self.assertNotEqual(-1, destroyed_at) + + after_states = ran_states[destroyed_at:] + self.assertGreater(0, len(after_states)) + + def test_claim_lost_new_owner(self): + job = self._post_claim_job('test') + f = self._make_dummy_flow(10) + e = self._make_engine(f) + + change_owner = True + ran_states = [] + with claims.CheckingClaimListener(e, job, + self.board, self.board.name): + for state in e.run_iter(): + ran_states.append(state) + if state == states.SCHEDULING and change_owner: + change_owner = bool(self._change_owner('test-2')) + + self.assertEqual(states.SUSPENDED, e.storage.get_flow_state()) + self.assertEqual(1, ran_states.count(states.ANALYZING)) + self.assertEqual(1, ran_states.count(states.SCHEDULING)) + self.assertEqual(1, ran_states.count(states.WAITING)) + + class TestTimingListener(test.TestCase, EngineMakerMixin): def test_duration(self): with contextlib.closing(impl_memory.MemoryBackend()) as be: From 14ecaa45503c8a0efcb5972f342de171b4b8cd9b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 1 Dec 2014 21:44:45 -0800 Subject: [PATCH 117/240] Raise value errors instead of asserts Since asserts can be silenced using -0 and/or compiled out we want to be more strict here and raise value errors when the expected callable objects/functions are not actually callable. Change-Id: Ib14e2e7329dbfcce50660f144ad4780d99f36854 --- taskflow/engines/worker_based/dispatcher.py | 3 ++- taskflow/task.py | 15 +++++++++------ taskflow/tests/unit/test_notifier.py | 5 +++++ taskflow/tests/unit/test_task.py | 11 +++++++++++ taskflow/tests/unit/test_types.py | 4 ++-- taskflow/types/fsm.py | 9 ++++++--- taskflow/types/notifier.py | 5 +++-- 7 files changed, 38 insertions(+), 14 deletions(-) diff --git a/taskflow/engines/worker_based/dispatcher.py b/taskflow/engines/worker_based/dispatcher.py index 1733df80..f354ec4b 100644 --- a/taskflow/engines/worker_based/dispatcher.py +++ b/taskflow/engines/worker_based/dispatcher.py @@ -42,7 +42,8 @@ class TypeDispatcher(object): filter should return a truthy object if the message should be requeued and a falsey object if it should not. """ - assert six.callable(callback), "Callback must be callable" + if not six.callable(callback): + raise ValueError("Requeue filter callback must be callable") self._requeue_filters.append(callback) def _collect_requeue_votes(self, data, message): diff --git a/taskflow/task.py b/taskflow/task.py index 7b2ec333..c08de07c 100644 --- a/taskflow/task.py +++ b/taskflow/task.py @@ -187,7 +187,8 @@ class BaseTask(atom.Atom): if event not in self.TASK_EVENTS: raise ValueError("Unknown task event '%s', can only bind" " to events %s" % (event, self.TASK_EVENTS)) - assert six.callable(handler), "Handler must be callable" + if not six.callable(handler): + raise ValueError("Event handler callback must be callable") self._events_listeners[event].append((handler, kwargs)) def unbind(self, event, handler=None): @@ -246,11 +247,13 @@ class FunctorTask(BaseTask): def __init__(self, execute, name=None, provides=None, requires=None, auto_extract=True, rebind=None, revert=None, version=None, inject=None): - assert six.callable(execute), ("Function to use for executing must be" - " callable") - if revert: - assert six.callable(revert), ("Function to use for reverting must" - " be callable") + if not six.callable(execute): + raise ValueError("Function to use for executing must be" + " callable") + if revert is not None: + if not six.callable(revert): + raise ValueError("Function to use for reverting must" + " be callable") if name is None: name = reflection.get_callable_name(execute) super(FunctorTask, self).__init__(name, provides=provides, diff --git a/taskflow/tests/unit/test_notifier.py b/taskflow/tests/unit/test_notifier.py index 0761cb6e..ab536077 100644 --- a/taskflow/tests/unit/test_notifier.py +++ b/taskflow/tests/unit/test_notifier.py @@ -79,6 +79,11 @@ class NotifierTest(test.TestCase): nt.Notifier.ANY, call_me, kwargs={'details': 5}) + def test_not_callable(self): + notifier = nt.Notifier() + self.assertRaises(ValueError, notifier.register, + nt.Notifier.ANY, 2) + def test_selective_notify(self): call_counts = collections.defaultdict(list) diff --git a/taskflow/tests/unit/test_task.py b/taskflow/tests/unit/test_task.py index a3854d26..8c9c7eff 100644 --- a/taskflow/tests/unit/test_task.py +++ b/taskflow/tests/unit/test_task.py @@ -282,6 +282,10 @@ class TaskTest(test.TestCase): self.assertFalse(task.unbind('update_progress', handler2)) self.assertEqual(len(task._events_listeners), 1) + def test_bind_not_callable(self): + task = MyTask() + self.assertRaises(ValueError, task.bind, 'update_progress', 2) + class FunctorTaskTest(test.TestCase): @@ -289,3 +293,10 @@ class FunctorTaskTest(test.TestCase): version = (2, 0) f_task = task.FunctorTask(lambda: None, version=version) self.assertEqual(f_task.version, version) + + def test_execute_not_callable(self): + self.assertRaises(ValueError, task.FunctorTask, 2) + + def test_revert_not_callable(self): + self.assertRaises(ValueError, task.FunctorTask, lambda: None, + revert=2) diff --git a/taskflow/tests/unit/test_types.py b/taskflow/tests/unit/test_types.py index 05dd07a1..395d7d9b 100644 --- a/taskflow/tests/unit/test_types.py +++ b/taskflow/tests/unit/test_types.py @@ -400,5 +400,5 @@ class FSMTest(test.TestCase): m = fsm.FSM('working') m.add_state('working') m.add_state('broken') - self.assertRaises(AssertionError, m.add_state, 'b', on_enter=2) - self.assertRaises(AssertionError, m.add_state, 'b', on_exit=2) + self.assertRaises(ValueError, m.add_state, 'b', on_enter=2) + self.assertRaises(ValueError, m.add_state, 'b', on_exit=2) diff --git a/taskflow/types/fsm.py b/taskflow/types/fsm.py index 99110508..edc9a57a 100644 --- a/taskflow/types/fsm.py +++ b/taskflow/types/fsm.py @@ -101,9 +101,11 @@ class FSM(object): if state in self._states: raise excp.Duplicate("State '%s' already defined" % state) if on_enter is not None: - assert six.callable(on_enter), "On enter callback must be callable" + if not six.callable(on_enter): + raise ValueError("On enter callback must be callable") if on_exit is not None: - assert six.callable(on_exit), "On exit callback must be callable" + if not six.callable(on_exit): + raise ValueError("On exit callback must be callable") self._states[state] = { 'terminal': bool(terminal), 'reactions': {}, @@ -137,7 +139,8 @@ class FSM(object): if state not in self._states: raise excp.NotFound("Can not add a reaction to event '%s' for an" " undefined state '%s'" % (event, state)) - assert six.callable(reaction), "Reaction callback must be callable" + if not six.callable(reaction): + raise ValueError("Reaction callback must be callable") if event not in self._states[state]['reactions']: self._states[state]['reactions'][event] = (reaction, args, kwargs) else: diff --git a/taskflow/types/notifier.py b/taskflow/types/notifier.py index a92d6b89..b8ce8c5f 100644 --- a/taskflow/types/notifier.py +++ b/taskflow/types/notifier.py @@ -99,9 +99,10 @@ class Notifier(object): ``details``, that will hold event details provided to the :meth:`.notify` method. """ - assert six.callable(callback), "Callback must be callable" + if not six.callable(callback): + raise ValueError("Notification callback must be callable") if self.is_registered(event_type, callback): - raise ValueError("Callback %s already registered" % (callback)) + raise ValueError("Notification callback already registered") if kwargs: for k in self.RESERVED_KEYS: if k in kwargs: From a69213821c34579ec224fa9a145ccc94d6bfa324 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 25 Nov 2014 15:40:07 -0800 Subject: [PATCH 118/240] Cache immutable visible scopes in the runtime component Instead of recalculating/rewalking over the visible scopes of atoms just calculate the scope once and cache it (as it currently does not change at runtime) and return the cached tuple instead to avoid the needless recreation whenever a scope is requested for a given atom. Change-Id: I47d24054c63e8620d26e7ade4baa239295daed0a --- taskflow/engines/action_engine/runtime.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/taskflow/engines/action_engine/runtime.py b/taskflow/engines/action_engine/runtime.py index de092f5e..ec0b44d4 100644 --- a/taskflow/engines/action_engine/runtime.py +++ b/taskflow/engines/action_engine/runtime.py @@ -40,6 +40,7 @@ class Runtime(object): self._task_executor = task_executor self._storage = storage self._compilation = compilation + self._scopes = {} @property def compilation(self): @@ -68,17 +69,23 @@ class Runtime(object): @misc.cachedproperty def retry_action(self): return ra.RetryAction(self._storage, self._atom_notifier, - lambda atom: sc.ScopeWalker(self.compilation, - atom, - names_only=True)) + self._fetch_scopes_for) @misc.cachedproperty def task_action(self): return ta.TaskAction(self._storage, self._task_executor, - self._atom_notifier, - lambda atom: sc.ScopeWalker(self.compilation, - atom, - names_only=True)) + self._atom_notifier, self._fetch_scopes_for) + + def _fetch_scopes_for(self, atom): + """Fetches a tuple of the visible scopes for the given atom.""" + try: + return self._scopes[atom] + except KeyError: + walker = sc.ScopeWalker(self.compilation, atom, + names_only=True) + visible_to = tuple(walker) + self._scopes[atom] = visible_to + return visible_to # Various helper methods used by the runtime components; not for public # consumption... From 6a6aa795fe9e3fa95599050a18a40bc99da42cc8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 3 Dec 2014 16:13:28 -0800 Subject: [PATCH 119/240] Some package additions and adjustments to the env_builder.sh This just adjusts to install wget and certain other packages before further work commences to ensure they are ready and avaiable for further use. Tested on rhel6.x as a cloud-init userdata script and it appears to continue to work as expected. Change-Id: Ia85715e27bc8342fd61d23e2c8a659dbce20aea9 --- tools/env_builder.sh | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/tools/env_builder.sh b/tools/env_builder.sh index b6171c7d..7adb39e9 100644 --- a/tools/env_builder.sh +++ b/tools/env_builder.sh @@ -30,6 +30,23 @@ Box () { echo } +Box "Installing system packages..." +if [ -f "/etc/redhat-release" ]; then + yum install -y -q mysql-devel postgresql-devel mysql-server \ + wget gcc make autoconf + mysqld="mysqld" + zookeeperd="zookeeper-server" +elif [ -f "/etc/debian_version" ]; then + apt-get -y -qq install libmysqlclient-dev mysql-server postgresql \ + wget gcc make autoconf + mysqld="mysql" + zookeeperd="zookeeper" +else + echo "Unknown distribution!!" + lsb_release -a + exit 1 +fi + set +e python_27=`which python2.7` set -e @@ -70,19 +87,6 @@ fi Box "Installing tox..." $pip_27 install -q 'tox>=1.6.1,<1.7.0' -Box "Installing system packages..." -if [ -f "/etc/redhat-release" ]; then - yum install -y -q mysql-devel postgresql-devel mysql-server - mysqld="mysqld" -elif [ -f "/etc/debian_version" ]; then - apt-get -y -qq install libmysqlclient-dev mysql-server postgresql - mysqld="mysql" -else - echo "Unknown distribution!!" - lsb_release -a - exit 1 -fi - Box "Setting up mysql..." service $mysqld restart /usr/bin/mysql --user="root" --execute='CREATE DATABASE 'openstack_citest'' @@ -95,10 +99,13 @@ FLUSH PRIVILEGES; EOF /usr/bin/mysql --user="root" < $build_dir/mysql.sql +# TODO(harlowja): configure/setup postgresql... + Box "Installing zookeeper..." -zk_file="cloudera-cdh-4-0.x86_64.rpm" -zk_url="http://archive.cloudera.com/cdh4/one-click-install/redhat/6/x86_64/$zk_file" if [ -f "/etc/redhat-release" ]; then + # RH doesn't ship zookeeper (still...) + zk_file="cloudera-cdh-4-0.x86_64.rpm" + zk_url="http://archive.cloudera.com/cdh4/one-click-install/redhat/6/x86_64/$zk_file" wget $zk_url -O $build_dir/$zk_file --no-check-certificate -nv yum -y -q --nogpgcheck localinstall $build_dir/$zk_file yum -y -q install zookeeper-server java @@ -106,10 +113,8 @@ if [ -f "/etc/redhat-release" ]; then service zookeeper-server init --force mkdir -pv /var/lib/zookeeper python -c "import random; print random.randint(1, 16384)" > /var/lib/zookeeper/myid - zookeeperd="zookeeper-server" elif [ -f "/etc/debian_version" ]; then apt-get install -y -qq zookeeperd - zookeeperd="zookeeper" else echo "Unknown distribution!!" lsb_release -a From c69884209a73792e85c2e8d2c5b1fab8685847cd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 29 Aug 2014 14:48:22 -0700 Subject: [PATCH 120/240] Be explicit about publish keyword arguments Instead of allowing for arbitrary keyword arguments which makes the API hard to change in the future prefer to have explicit arguments (and keyword arguments) instead of allowing **kwargs to be passed. Change-Id: I374db6b19ef76c2f9ee04771f5d928c79b7cf049 --- taskflow/engines/worker_based/executor.py | 3 +-- taskflow/engines/worker_based/proxy.py | 20 +++++++++---------- .../tests/unit/worker_based/test_executor.py | 12 +++++------ .../tests/unit/worker_based/test_proxy.py | 11 +++++----- 4 files changed, 22 insertions(+), 24 deletions(-) diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index ae8e0e40..5982a463 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -211,8 +211,7 @@ class WorkerTaskExecutor(executor.TaskExecutorBase): " correlation_id=%s)", request, topic, self._uuid, request.uuid) try: - self._proxy.publish(msg=request, - routing_key=topic, + self._proxy.publish(request, topic, reply_to=self._uuid, correlation_id=request.uuid) except Exception: diff --git a/taskflow/engines/worker_based/proxy.py b/taskflow/engines/worker_based/proxy.py index 6f608f42..3f279a77 100644 --- a/taskflow/engines/worker_based/proxy.py +++ b/taskflow/engines/worker_based/proxy.py @@ -99,16 +99,15 @@ class Proxy(object): """Return whether the proxy is running.""" return self._running.is_set() - def _make_queue(self, name, exchange, **kwargs): - """Make named queue for the given exchange.""" - return kombu.Queue(name="%s_%s" % (self._exchange_name, name), - exchange=exchange, - routing_key=name, - durable=False, - auto_delete=True, - **kwargs) + def _make_queue(self, routing_key, exchange, channel=None): + """Make a named queue for the given exchange.""" + queue_name = "%s_%s" % (self._exchange_name, routing_key) + return kombu.Queue(name=queue_name, + routing_key=routing_key, durable=False, + exchange=exchange, auto_delete=True, + channel=channel) - def publish(self, msg, routing_key, **kwargs): + def publish(self, msg, routing_key, reply_to=None, correlation_id=None): """Publish message to the named exchange with given routing key.""" if isinstance(routing_key, six.string_types): routing_keys = [routing_key] @@ -123,7 +122,8 @@ class Proxy(object): exchange=self._exchange, declare=[queue], type=msg.TYPE, - **kwargs) + reply_to=reply_to, + correlation_id=correlation_id) def start(self): """Start proxy.""" diff --git a/taskflow/tests/unit/worker_based/test_executor.py b/taskflow/tests/unit/worker_based/test_executor.py index d2b97bfe..ba73ad7e 100644 --- a/taskflow/tests/unit/worker_based/test_executor.py +++ b/taskflow/tests/unit/worker_based/test_executor.py @@ -214,8 +214,8 @@ class TestWorkerTaskExecutor(test.MockTestCase): self.task_args, None, self.timeout), mock.call.request.transition_and_log_error(pr.PENDING, logger=mock.ANY), - mock.call.proxy.publish(msg=self.request_inst_mock, - routing_key=self.executor_topic, + mock.call.proxy.publish(self.request_inst_mock, + self.executor_topic, reply_to=self.executor_uuid, correlation_id=self.task_uuid) ] @@ -236,8 +236,8 @@ class TestWorkerTaskExecutor(test.MockTestCase): result=self.task_result), mock.call.request.transition_and_log_error(pr.PENDING, logger=mock.ANY), - mock.call.proxy.publish(msg=self.request_inst_mock, - routing_key=self.executor_topic, + mock.call.proxy.publish(self.request_inst_mock, + self.executor_topic, reply_to=self.executor_uuid, correlation_id=self.task_uuid) ] @@ -267,8 +267,8 @@ class TestWorkerTaskExecutor(test.MockTestCase): self.task_args, None, self.timeout), mock.call.request.transition_and_log_error(pr.PENDING, logger=mock.ANY), - mock.call.proxy.publish(msg=self.request_inst_mock, - routing_key=self.executor_topic, + mock.call.proxy.publish(self.request_inst_mock, + self.executor_topic, reply_to=self.executor_uuid, correlation_id=self.task_uuid), mock.call.request.transition_and_log_error(pr.FAILURE, diff --git a/taskflow/tests/unit/worker_based/test_proxy.py b/taskflow/tests/unit/worker_based/test_proxy.py index 4217a726..a3c7d13f 100644 --- a/taskflow/tests/unit/worker_based/test_proxy.py +++ b/taskflow/tests/unit/worker_based/test_proxy.py @@ -16,10 +16,9 @@ import socket -from six.moves import mock - from taskflow.engines.worker_based import proxy from taskflow import test +from taskflow.test import mock from taskflow.utils import threading_utils @@ -133,24 +132,24 @@ class TestProxy(test.MockTestCase): msg_mock.to_dict.return_value = msg_data routing_key = 'routing-key' task_uuid = 'task-uuid' - kwargs = dict(a='a', b='b') self.proxy(reset_master_mock=True).publish( - msg_mock, routing_key, correlation_id=task_uuid, **kwargs) + msg_mock, routing_key, correlation_id=task_uuid) master_mock_calls = [ mock.call.Queue(name=self._queue_name(routing_key), exchange=self.exchange_inst_mock, routing_key=routing_key, durable=False, - auto_delete=True), + auto_delete=True, + channel=None), mock.call.producer.publish(body=msg_data, routing_key=routing_key, exchange=self.exchange_inst_mock, correlation_id=task_uuid, declare=[self.queue_inst_mock], type=msg_mock.TYPE, - **kwargs) + reply_to=None) ] self.master_mock.assert_has_calls(master_mock_calls) From cf45a7045921b1109cdb11dd2f7d374791d20a37 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 24 Nov 2014 21:28:37 -0800 Subject: [PATCH 121/240] Allow for the notifier to provide a 'details_filter' When a notifier is being used for retry atoms and for task atoms it is useful to allow a filter callback to be provided that will skip notification depending on whether the details matches or not (so that task or retry notifications can proceed). Part of fix for bug 1395966 Change-Id: If03005c253c3f540d2c33faf7a7474a5fde9dcdd --- taskflow/tests/unit/test_notifier.py | 74 +++++++++++++++-- taskflow/types/notifier.py | 119 +++++++++++++++++++-------- 2 files changed, 154 insertions(+), 39 deletions(-) diff --git a/taskflow/tests/unit/test_notifier.py b/taskflow/tests/unit/test_notifier.py index ab536077..d9d40001 100644 --- a/taskflow/tests/unit/test_notifier.py +++ b/taskflow/tests/unit/test_notifier.py @@ -91,11 +91,16 @@ class NotifierTest(test.TestCase): call_counts[registered_state].append((state, details)) notifier = nt.Notifier() - notifier.register(states.SUCCESS, - functools.partial(call_me_on, states.SUCCESS)) - notifier.register(nt.Notifier.ANY, - functools.partial(call_me_on, - nt.Notifier.ANY)) + + call_me_on_success = functools.partial(call_me_on, states.SUCCESS) + notifier.register(states.SUCCESS, call_me_on_success) + self.assertTrue(notifier.is_registered(states.SUCCESS, + call_me_on_success)) + + call_me_on_any = functools.partial(call_me_on, nt.Notifier.ANY) + notifier.register(nt.Notifier.ANY, call_me_on_any) + self.assertTrue(notifier.is_registered(nt.Notifier.ANY, + call_me_on_any)) self.assertEqual(2, len(notifier)) notifier.notify(states.SUCCESS, {}) @@ -107,3 +112,62 @@ class NotifierTest(test.TestCase): self.assertEqual(2, len(call_counts[nt.Notifier.ANY])) self.assertEqual(1, len(call_counts[states.SUCCESS])) self.assertEqual(2, len(call_counts)) + + def test_details_filter(self): + call_counts = collections.defaultdict(list) + + def call_me_on(registered_state, state, details): + call_counts[registered_state].append((state, details)) + + def when_red(details): + return details.get('color') == 'red' + + notifier = nt.Notifier() + + call_me_on_success = functools.partial(call_me_on, states.SUCCESS) + notifier.register(states.SUCCESS, call_me_on_success, + details_filter=when_red) + self.assertEqual(1, len(notifier)) + self.assertTrue(notifier.is_registered( + states.SUCCESS, call_me_on_success, details_filter=when_red)) + + notifier.notify(states.SUCCESS, {}) + self.assertEqual(0, len(call_counts[states.SUCCESS])) + notifier.notify(states.SUCCESS, {'color': 'red'}) + self.assertEqual(1, len(call_counts[states.SUCCESS])) + notifier.notify(states.SUCCESS, {'color': 'green'}) + self.assertEqual(1, len(call_counts[states.SUCCESS])) + + def test_different_details_filter(self): + call_counts = collections.defaultdict(list) + + def call_me_on(registered_state, state, details): + call_counts[registered_state].append((state, details)) + + def when_red(details): + return details.get('color') == 'red' + + def when_blue(details): + return details.get('color') == 'blue' + + notifier = nt.Notifier() + + call_me_on_success = functools.partial(call_me_on, states.SUCCESS) + notifier.register(states.SUCCESS, call_me_on_success, + details_filter=when_red) + notifier.register(states.SUCCESS, call_me_on_success, + details_filter=when_blue) + self.assertEqual(2, len(notifier)) + self.assertTrue(notifier.is_registered( + states.SUCCESS, call_me_on_success, details_filter=when_blue)) + self.assertTrue(notifier.is_registered( + states.SUCCESS, call_me_on_success, details_filter=when_red)) + + notifier.notify(states.SUCCESS, {}) + self.assertEqual(0, len(call_counts[states.SUCCESS])) + notifier.notify(states.SUCCESS, {'color': 'red'}) + self.assertEqual(1, len(call_counts[states.SUCCESS])) + notifier.notify(states.SUCCESS, {'color': 'blue'}) + self.assertEqual(2, len(call_counts[states.SUCCESS])) + notifier.notify(states.SUCCESS, {'color': 'green'}) + self.assertEqual(2, len(call_counts[states.SUCCESS])) diff --git a/taskflow/types/notifier.py b/taskflow/types/notifier.py index b8ce8c5f..8e58302f 100644 --- a/taskflow/types/notifier.py +++ b/taskflow/types/notifier.py @@ -15,7 +15,6 @@ # under the License. import collections -import copy import logging import six @@ -25,6 +24,56 @@ from taskflow.utils import reflection LOG = logging.getLogger(__name__) +class _Listener(object): + """Internal helper that represents a notification listener/target.""" + + def __init__(self, callback, args=None, kwargs=None, details_filter=None): + self._callback = callback + self._details_filter = details_filter + if not args: + self._args = () + else: + self._args = args[:] + if not kwargs: + self._kwargs = {} + else: + self._kwargs = kwargs.copy() + + def __call__(self, event_type, details): + if self._details_filter is not None: + if not self._details_filter(details): + return + kwargs = self._kwargs.copy() + kwargs['details'] = details + self._callback(event_type, *self._args, **kwargs) + + def __repr__(self): + repr_msg = "%s object at 0x%x calling into '%r'" % ( + reflection.get_class_name(self), id(self), self._callback) + if self._details_filter is not None: + repr_msg += " using details filter '%r'" % self._details_filter + return "<%s>" % repr_msg + + def is_equivalent(self, callback, details_filter=None): + if not reflection.is_same_callback(self._callback, callback): + return False + if details_filter is not None: + if self._details_filter is None: + return False + else: + return reflection.is_same_callback(self._details_filter, + details_filter) + else: + return self._details_filter is None + + def __eq__(self, other): + if isinstance(other, _Listener): + return self.is_equivalent(other._callback, + details_filter=other._details_filter) + else: + return NotImplemented + + class Notifier(object): """A notification helper class. @@ -34,7 +83,7 @@ class Notifier(object): notification occurs. """ - #: Keys that can not be used in callbacks arguments + #: Keys that can *not* be used in callbacks arguments RESERVED_KEYS = ('details',) #: Kleene star constant that is used to recieve all notifications @@ -46,15 +95,14 @@ class Notifier(object): def __len__(self): """Returns how many callbacks are registered.""" count = 0 - for (_event_type, callbacks) in six.iteritems(self._listeners): - count += len(callbacks) + for (_event_type, listeners) in six.iteritems(self._listeners): + count += len(listeners) return count - def is_registered(self, event_type, callback): + def is_registered(self, event_type, callback, details_filter=None): """Check if a callback is registered.""" - listeners = list(self._listeners.get(event_type, [])) - for (cb, _args, _kwargs) in listeners: - if reflection.is_same_callback(cb, callback): + for listener in self._listeners.get(event_type, []): + if listener.is_equivalent(callback, details_filter=details_filter): return True return False @@ -72,52 +120,55 @@ class Notifier(object): :param details: addition event details """ listeners = list(self._listeners.get(self.ANY, [])) - for i in self._listeners[event_type]: - if i not in listeners: - listeners.append(i) + for listener in self._listeners[event_type]: + if listener not in listeners: + listeners.append(listener) if not listeners: return - for (callback, args, kwargs) in listeners: - if args is None: - args = [] - if kwargs is None: - kwargs = {} - kwargs['details'] = details + for listener in listeners: try: - callback(event_type, *args, **kwargs) + listener(event_type, details) except Exception: - LOG.warn("Failure calling callback %s to notify about event" - " %s, details: %s", callback, event_type, + LOG.warn("Failure calling listener %s to notify about event" + " %s, details: %s", listener, event_type, details, exc_info=True) - def register(self, event_type, callback, args=None, kwargs=None): + def register(self, event_type, callback, + args=None, kwargs=None, details_filter=None): """Register a callback to be called when event of a given type occurs. Callback will be called with provided ``args`` and ``kwargs`` and when event type occurs (or on any event if ``event_type`` equals to :attr:`.ANY`). It will also get additional keyword argument, ``details``, that will hold event details provided to the - :meth:`.notify` method. + :meth:`.notify` method (if a details filter callback is provided then + the target callback will *only* be triggered if the details filter + callback returns a truthy value). """ if not six.callable(callback): - raise ValueError("Notification callback must be callable") - if self.is_registered(event_type, callback): - raise ValueError("Notification callback already registered") + raise ValueError("Event callback must be callable") + if details_filter is not None: + if not six.callable(details_filter): + raise ValueError("Details filter must be callable") + if self.is_registered(event_type, callback, + details_filter=details_filter): + raise ValueError("Event callback already registered with" + " equivalent details filter") if kwargs: for k in self.RESERVED_KEYS: if k in kwargs: - raise KeyError(("Reserved key '%s' not allowed in " - "kwargs") % k) - kwargs = copy.copy(kwargs) - if args: - args = copy.copy(args) - self._listeners[event_type].append((callback, args, kwargs)) + raise KeyError("Reserved key '%s' not allowed in " + "kwargs" % k) + self._listeners[event_type].append( + _Listener(callback, + args=args, kwargs=kwargs, + details_filter=details_filter)) - def deregister(self, event_type, callback): + def deregister(self, event_type, callback, details_filter=None): """Remove a single callback from listening to event ``event_type``.""" if event_type not in self._listeners: return - for i, (cb, args, kwargs) in enumerate(self._listeners[event_type]): - if reflection.is_same_callback(cb, callback): + for i, listener in enumerate(self._listeners[event_type]): + if listener.is_equivalent(callback, details_filter=details_filter): self._listeners[event_type].pop(i) break From b8e975e885e4f5aaab52edcd23fee99d450273ad Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 24 Nov 2014 19:44:52 -0800 Subject: [PATCH 122/240] Update listeners to ensure they correctly handle all atoms Instead of looking for keys that are task specific (as well as using the deprecated 'task_notifier') we need to update the listeners to be agnostic to atoms (retry or task) that are sent to them so that key errors do not occur when extracting any data sent along with the event notification. Fixes bug 1395966 Change-Id: Ib61b34b83203f5999f92b6e8616efd90cb259f81 --- taskflow/listeners/base.py | 125 ++++++++++++++++++++++----------- taskflow/listeners/logging.py | 13 ++-- taskflow/listeners/printing.py | 7 +- 3 files changed, 95 insertions(+), 50 deletions(-) diff --git a/taskflow/listeners/base.py b/taskflow/listeners/base.py index 739db49c..f69bb87e 100644 --- a/taskflow/listeners/base.py +++ b/taskflow/listeners/base.py @@ -36,32 +36,80 @@ FINISH_STATES = (states.FAILURE, states.SUCCESS) DEFAULT_LISTEN_FOR = (notifier.Notifier.ANY,) +def _task_matcher(details): + """Matches task details emitted.""" + if not details: + return False + if 'task_name' in details and 'task_uuid' in details: + return True + return False + + +def _retry_matcher(details): + """Matches retry details emitted.""" + if not details: + return False + if 'retry_name' in details and 'retry_uuid' in details: + return True + return False + + +def _bulk_deregister(notifier, registered, details_filter=None): + """Bulk deregisters callbacks associated with many states.""" + while registered: + state, cb = registered.pop() + notifier.deregister(state, cb, + details_filter=details_filter) + + +def _bulk_register(watch_states, notifier, cb, details_filter=None): + """Bulk registers a callback associated with many states.""" + registered = [] + try: + for state in watch_states: + if not notifier.is_registered(state, cb, + details_filter=details_filter): + notifier.register(state, cb, + details_filter=details_filter) + registered.append((state, cb)) + except ValueError: + with excutils.save_and_reraise_exception(): + _bulk_deregister(notifier, registered, + details_filter=details_filter) + else: + return registered + + class ListenerBase(object): """Base class for listeners. A listener can be attached to an engine to do various actions on flow and - task state transitions. It implements context manager protocol to be able - to register and unregister with a given engine automatically when a context - is entered and when it is exited. + atom state transitions. It implements the context manager protocol to be + able to register and unregister with a given engine automatically when a + context is entered and when it is exited. To implement a listener, derive from this class and override - ``_flow_receiver`` and/or ``_task_receiver`` methods (in this class, - they do nothing). + ``_flow_receiver`` and/or ``_task_receiver`` and/or ``_retry_receiver`` + methods (in this class, they do nothing). """ def __init__(self, engine, task_listen_for=DEFAULT_LISTEN_FOR, - flow_listen_for=DEFAULT_LISTEN_FOR): + flow_listen_for=DEFAULT_LISTEN_FOR, + retry_listen_for=DEFAULT_LISTEN_FOR): if not task_listen_for: task_listen_for = [] + if not retry_listen_for: + retry_listen_for = [] if not flow_listen_for: flow_listen_for = [] self._listen_for = { 'task': list(task_listen_for), + 'retry': list(retry_listen_for), 'flow': list(flow_listen_for), } self._engine = engine - self._registered = False + self._registered = {} def _flow_receiver(self, state, details): pass @@ -69,43 +117,38 @@ class ListenerBase(object): def _task_receiver(self, state, details): pass + def _retry_receiver(self, state, details): + pass + def deregister(self): - if not self._registered: - return - - def _deregister(watch_states, notifier, cb): - for s in watch_states: - notifier.deregister(s, cb) - - _deregister(self._listen_for['task'], self._engine.task_notifier, - self._task_receiver) - _deregister(self._listen_for['flow'], self._engine.notifier, - self._flow_receiver) - - self._registered = False + if 'task' in self._registered: + _bulk_deregister(self._engine.atom_notifier, + self._registered['task'], + details_filter=_task_matcher) + del self._registered['task'] + if 'retry' in self._registered: + _bulk_deregister(self._engine.atom_notifier, + self._registered['retry'], + details_filter=_retry_matcher) + del self._registered['retry'] + if 'flow' in self._registered: + _bulk_deregister(self._engine.notifier, + self._registered['flow']) + del self._registered['flow'] def register(self): - if self._registered: - return - - def _register(watch_states, notifier, cb): - registered = [] - try: - for s in watch_states: - if not notifier.is_registered(s, cb): - notifier.register(s, cb) - registered.append((s, cb)) - except ValueError: - with excutils.save_and_reraise_exception(): - for (s, cb) in registered: - notifier.deregister(s, cb) - - _register(self._listen_for['task'], self._engine.task_notifier, - self._task_receiver) - _register(self._listen_for['flow'], self._engine.notifier, - self._flow_receiver) - - self._registered = True + if 'task' not in self._registered: + self._registered['task'] = _bulk_register( + self._listen_for['task'], self._engine.atom_notifier, + self._task_receiver, details_filter=_task_matcher) + if 'retry' not in self._registered: + self._registered['retry'] = _bulk_register( + self._listen_for['retry'], self._engine.atom_notifier, + self._retry_receiver, details_filter=_retry_matcher) + if 'flow' not in self._registered: + self._registered['flow'] = _bulk_register( + self._listen_for['flow'], self._engine.notifier, + self._flow_receiver) def __enter__(self): self.register() diff --git a/taskflow/listeners/logging.py b/taskflow/listeners/logging.py index 20ff1baa..51bf693c 100644 --- a/taskflow/listeners/logging.py +++ b/taskflow/listeners/logging.py @@ -53,11 +53,12 @@ class LoggingListener(base.LoggingBase): def __init__(self, engine, task_listen_for=base.DEFAULT_LISTEN_FOR, flow_listen_for=base.DEFAULT_LISTEN_FOR, + retry_listen_for=base.DEFAULT_LISTEN_FOR, log=None, level=logging.DEBUG): - super(LoggingListener, self).__init__(engine, - task_listen_for=task_listen_for, - flow_listen_for=flow_listen_for) + super(LoggingListener, self).__init__( + engine, task_listen_for=task_listen_for, + flow_listen_for=flow_listen_for, retry_listen_for=retry_listen_for) if not log: self._logger = LOG else: @@ -101,12 +102,12 @@ class DynamicLoggingListener(base.ListenerBase): def __init__(self, engine, task_listen_for=base.DEFAULT_LISTEN_FOR, flow_listen_for=base.DEFAULT_LISTEN_FOR, + retry_listen_for=base.DEFAULT_LISTEN_FOR, log=None, failure_level=logging.WARNING, level=logging.DEBUG): super(DynamicLoggingListener, self).__init__( - engine, - task_listen_for=task_listen_for, - flow_listen_for=flow_listen_for) + engine, task_listen_for=task_listen_for, + flow_listen_for=flow_listen_for, retry_listen_for=retry_listen_for) self._failure_level = failure_level self._level = level self._task_log_levels = { diff --git a/taskflow/listeners/printing.py b/taskflow/listeners/printing.py index 397914ed..719d2042 100644 --- a/taskflow/listeners/printing.py +++ b/taskflow/listeners/printing.py @@ -27,10 +27,11 @@ class PrintingListener(base.LoggingBase): def __init__(self, engine, task_listen_for=base.DEFAULT_LISTEN_FOR, flow_listen_for=base.DEFAULT_LISTEN_FOR, + retry_listen_for=base.DEFAULT_LISTEN_FOR, stderr=False): - super(PrintingListener, self).__init__(engine, - task_listen_for=task_listen_for, - flow_listen_for=flow_listen_for) + super(PrintingListener, self).__init__( + engine, task_listen_for=task_listen_for, + flow_listen_for=flow_listen_for, retry_listen_for=retry_listen_for) if stderr: self._file = sys.stderr else: From 81505533675d3edff88ab0dc1ac04b0a558bbbbe Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 4 Dec 2014 16:11:56 -0800 Subject: [PATCH 123/240] Fix split on "+" for connection strings that specify dialects Fixes bug 1399486 Change-Id: I3b7e6331751f25d9c4221393e8329934925791e7 --- taskflow/persistence/backends/__init__.py | 5 +++++ .../unit/persistence/test_sql_persistence.py | 16 ++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/taskflow/persistence/backends/__init__.py b/taskflow/persistence/backends/__init__.py index 64b7cda1..30279b90 100644 --- a/taskflow/persistence/backends/__init__.py +++ b/taskflow/persistence/backends/__init__.py @@ -58,6 +58,11 @@ def fetch(conf, namespace=BACKEND_NAMESPACE, **kwargs): else: backend_name = uri.scheme conf = misc.merge_uri(uri, conf.copy()) + # If the backend is like 'mysql+pymysql://...' which informs the + # backend to use a dialect (supported by sqlalchemy at least) we just want + # to look at the first component to find our entrypoint backend name... + if backend_name.find("+") != -1: + backend_name = backend_name.split("+", 1)[0] LOG.debug('Looking for %r backend driver in %r', backend_name, namespace) try: mgr = driver.DriverManager(namespace, backend_name, diff --git a/taskflow/tests/unit/persistence/test_sql_persistence.py b/taskflow/tests/unit/persistence/test_sql_persistence.py index 229ef310..8489160d 100644 --- a/taskflow/tests/unit/persistence/test_sql_persistence.py +++ b/taskflow/tests/unit/persistence/test_sql_persistence.py @@ -145,6 +145,14 @@ class BackendPersistenceTestMixin(base.PersistenceTestMixin): def _get_connection(self): return self.backend.get_connection() + def test_entrypoint(self): + # Test that the entrypoint fetching also works (even with dialects) + # using the same configuration we used in setUp() but not using + # the impl_sqlalchemy SQLAlchemyBackend class directly... + with contextlib.closing(backends.fetch(self.db_conf)) as backend: + with contextlib.closing(backend.get_connection()): + pass + @abc.abstractmethod def _init_db(self): """Sets up the database, and returns the uri to that database.""" @@ -158,17 +166,17 @@ class BackendPersistenceTestMixin(base.PersistenceTestMixin): self.backend = None try: self.db_uri = self._init_db() + self.db_conf = { + 'connection': self.db_uri + } # Since we are using random database names, we need to make sure # and remove our random database when we are done testing. self.addCleanup(self._remove_db) - conf = { - 'connection': self.db_uri - } except Exception as e: self.skipTest("Failed to create temporary database;" " testing being skipped due to: %s" % (e)) try: - self.backend = impl_sqlalchemy.SQLAlchemyBackend(conf) + self.backend = impl_sqlalchemy.SQLAlchemyBackend(self.db_conf) self.addCleanup(self.backend.close) with contextlib.closing(self._get_connection()) as conn: conn.upgrade() From 2f037360d6d2d4eaf803f8d8f9374da2d980d68f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 4 Dec 2014 17:40:13 -0800 Subject: [PATCH 124/240] Ensure frozen attribute is set in fsm clones/copies Change-Id: Ic267eaeeda70046e37e6edeb296a956f32c82edf --- taskflow/types/fsm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/taskflow/types/fsm.py b/taskflow/types/fsm.py index edc9a57a..9cf94d7b 100644 --- a/taskflow/types/fsm.py +++ b/taskflow/types/fsm.py @@ -209,6 +209,7 @@ class FSM(object): NOTE(harlowja): the copy will be left in an *uninitialized* state. """ c = FSM(self.start_state) + c.frozen = self.frozen for state, data in six.iteritems(self._states): copied_data = data.copy() copied_data['reactions'] = copied_data['reactions'].copy() From 2033d011e3cdc320a829ded3288c8d6b7f34aaff Mon Sep 17 00:00:00 2001 From: Jeremy Stanley Date: Fri, 5 Dec 2014 03:30:41 +0000 Subject: [PATCH 125/240] Workflow documentation is now in infra-manual Replace URLs for workflow documentation to appropriate parts of the OpenStack Project Infrastructure Manual. Change-Id: I29028bae8800e0e5cc4e3c0ec56756f82f0e6fe3 --- CONTRIBUTING.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 988f2856..2762e800 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,13 +1,13 @@ If you would like to contribute to the development of OpenStack, you must follow the steps documented at: - http://wiki.openstack.org/HowToContribute#If_you.27re_a_developer + http://docs.openstack.org/infra/manual/developers.html#development-workflow Once those steps have been completed, changes to OpenStack should be submitted for review via the Gerrit tool, following the workflow documented at: - http://wiki.openstack.org/GerritWorkflow + http://docs.openstack.org/infra/manual/developers.html#development-workflow Pull requests submitted through GitHub will be ignored. From 4707ac74e15cda42fb73199e04f6f5868605a323 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 4 Dec 2014 23:59:54 -0800 Subject: [PATCH 126/240] Move atom action handlers to there own subfolder/submodule Since action handlers are atom specific it seems appropriate to place the handlers under a specific submodule so that the action engine module/folder layout is more obvious and makes more sense. This also adds on a handles(atom) method to those handlers so that the runtime module/class can use those methods to determine which handler should handle resetting an atom to its initial state. Change-Id: I7067d91347b41613fba1492d81d68a590f22c467 --- taskflow/engines/action_engine/actions/__init__.py | 0 .../{retry_action.py => actions/retry.py} | 12 +++++++++--- .../{task_action.py => actions/task.py} | 6 ++++++ taskflow/engines/action_engine/runtime.py | 13 ++++++------- 4 files changed, 21 insertions(+), 10 deletions(-) create mode 100644 taskflow/engines/action_engine/actions/__init__.py rename taskflow/engines/action_engine/{retry_action.py => actions/retry.py} (92%) rename taskflow/engines/action_engine/{task_action.py => actions/task.py} (96%) diff --git a/taskflow/engines/action_engine/actions/__init__.py b/taskflow/engines/action_engine/actions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/taskflow/engines/action_engine/retry_action.py b/taskflow/engines/action_engine/actions/retry.py similarity index 92% rename from taskflow/engines/action_engine/retry_action.py rename to taskflow/engines/action_engine/actions/retry.py index df710292..5afd2751 100644 --- a/taskflow/engines/action_engine/retry_action.py +++ b/taskflow/engines/action_engine/actions/retry.py @@ -17,7 +17,7 @@ import logging from taskflow.engines.action_engine import executor as ex -from taskflow import retry as rt +from taskflow import retry as retry_atom from taskflow import states from taskflow.types import failure from taskflow.types import futures @@ -28,19 +28,25 @@ SAVE_RESULT_STATES = (states.SUCCESS, states.FAILURE) class RetryAction(object): + """An action that handles executing, state changes, ... of retry atoms.""" + def __init__(self, storage, notifier, walker_factory): self._storage = storage self._notifier = notifier self._walker_factory = walker_factory self._executor = futures.SynchronousExecutor() + @staticmethod + def handles(atom): + return isinstance(atom, retry_atom.Retry) + def _get_retry_args(self, retry, addons=None): scope_walker = self._walker_factory(retry) kwargs = self._storage.fetch_mapped_args(retry.rebind, atom_name=retry.name, scope_walker=scope_walker) history = self._storage.get_retry_history(retry.name) - kwargs[rt.EXECUTE_REVERT_HISTORY] = history + kwargs[retry_atom.EXECUTE_REVERT_HISTORY] = history if addons: kwargs.update(addons) return kwargs @@ -103,7 +109,7 @@ class RetryAction(object): self.change_state(retry, states.REVERTING) arg_addons = { - rt.REVERT_FLOW_FAILURES: self._storage.get_failures(), + retry_atom.REVERT_FLOW_FAILURES: self._storage.get_failures(), } fut = self._executor.submit(_execute_retry, self._get_retry_args(retry, diff --git a/taskflow/engines/action_engine/task_action.py b/taskflow/engines/action_engine/actions/task.py similarity index 96% rename from taskflow/engines/action_engine/task_action.py rename to taskflow/engines/action_engine/actions/task.py index eb9510f9..6c520f16 100644 --- a/taskflow/engines/action_engine/task_action.py +++ b/taskflow/engines/action_engine/actions/task.py @@ -17,6 +17,7 @@ import logging from taskflow import states +from taskflow import task as task_atom from taskflow.types import failure LOG = logging.getLogger(__name__) @@ -25,6 +26,7 @@ SAVE_RESULT_STATES = (states.SUCCESS, states.FAILURE) class TaskAction(object): + """An action that handles scheduling, state changes, ... of task atoms.""" def __init__(self, storage, task_executor, notifier, walker_factory): self._storage = storage @@ -32,6 +34,10 @@ class TaskAction(object): self._notifier = notifier self._walker_factory = walker_factory + @staticmethod + def handles(atom): + return isinstance(atom, task_atom.BaseTask) + def _is_identity_transition(self, state, task, progress): if state in SAVE_RESULT_STATES: # saving result is never identity transition diff --git a/taskflow/engines/action_engine/runtime.py b/taskflow/engines/action_engine/runtime.py index ec0b44d4..8f9b56b5 100644 --- a/taskflow/engines/action_engine/runtime.py +++ b/taskflow/engines/action_engine/runtime.py @@ -14,16 +14,14 @@ # License for the specific language governing permissions and limitations # under the License. +from taskflow.engines.action_engine.actions import retry as ra +from taskflow.engines.action_engine.actions import task as ta from taskflow.engines.action_engine import analyzer as an from taskflow.engines.action_engine import completer as co -from taskflow.engines.action_engine import retry_action as ra from taskflow.engines.action_engine import runner as ru from taskflow.engines.action_engine import scheduler as sched from taskflow.engines.action_engine import scopes as sc -from taskflow.engines.action_engine import task_action as ta -from taskflow import retry as retry_atom from taskflow import states as st -from taskflow import task as task_atom from taskflow.utils import misc @@ -93,9 +91,10 @@ class Runtime(object): def reset_nodes(self, nodes, state=st.PENDING, intention=st.EXECUTE): for node in nodes: if state: - if isinstance(node, task_atom.BaseTask): - self.task_action.change_state(node, state, progress=0.0) - elif isinstance(node, retry_atom.Retry): + if self.task_action.handles(node): + self.task_action.change_state(node, state, + progress=0.0) + elif self.retry_action.handles(node): self.retry_action.change_state(node, state) else: raise TypeError("Unknown how to reset atom '%s' (%s)" From dc393513b49320bb5cbaaa4c582db292ec3be796 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 5 Dec 2014 21:10:18 -0800 Subject: [PATCH 127/240] Just use 4 spaces for classifier indents Keep the setup.cfg file consistently using only four spaces and not having mixed 4 and 8 spaces. Change-Id: I2e77766e6491be91e6744d4acf84eac16e39c627 --- setup.cfg | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/setup.cfg b/setup.cfg index 660c2822..cfef68eb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,21 +9,21 @@ home-page = https://launchpad.net/taskflow keywords = reliable,tasks,execution,parallel,dataflow,workflows,distributed requires-python = >=2.6 classifier = - Development Status :: 4 - Beta - Environment :: OpenStack - Intended Audience :: Developers - Intended Audience :: Information Technology - License :: OSI Approved :: Apache Software License - Operating System :: POSIX :: Linux - Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.6 - Programming Language :: Python :: 2.7 - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.3 - Programming Language :: Python :: 3.4 - Topic :: Software Development :: Libraries - Topic :: System :: Distributed Computing + Development Status :: 4 - Beta + Environment :: OpenStack + Intended Audience :: Developers + Intended Audience :: Information Technology + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.6 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.3 + Programming Language :: Python :: 3.4 + Topic :: Software Development :: Libraries + Topic :: System :: Distributed Computing [global] setup-hooks = From e168f44979b34cef76a7c613a383f8b050091f33 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 19 Oct 2014 23:43:01 -0700 Subject: [PATCH 128/240] Rework pieces of the task callback capability Unifies the bind, unbind, autobind parameters. Also to make it easier to introspect what are a tasks associated callbacks and events are provide a listeners_iter() method that can be used to introspect the registered (event, callbacks) pairs that are registered with a task. Also adds more useful docstrings to the various callback associated binding, unbinding functions to make it more understandable how they are used and what they are provided. Also makes the currently only default provided event 'update_progress' a constant that can be referenced from the task module, which allows others to easily find it and use it. Change-Id: I14181a150b74fbd97f6ea976723f37c0ba4cec36 --- taskflow/engines/action_engine/executor.py | 4 +- taskflow/task.py | 134 ++++++++++++++------- taskflow/tests/unit/test_task.py | 72 +++++------ 3 files changed, 126 insertions(+), 84 deletions(-) diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index 78d16ff9..4c5c091a 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -30,7 +30,7 @@ REVERTED = 'reverted' def _execute_task(task, arguments, progress_callback): - with task.autobind('update_progress', progress_callback): + with task.autobind(_task.EVENT_UPDATE_PROGRESS, progress_callback): try: task.pre_execute() result = task.execute(**arguments) @@ -47,7 +47,7 @@ def _revert_task(task, arguments, result, failures, progress_callback): kwargs = arguments.copy() kwargs[_task.REVERT_RESULT] = result kwargs[_task.REVERT_FLOW_FAILURES] = failures - with task.autobind('update_progress', progress_callback): + with task.autobind(_task.EVENT_UPDATE_PROGRESS, progress_callback): try: task.pre_revert() result = task.revert(**kwargs) diff --git a/taskflow/task.py b/taskflow/task.py index 7c8df6cd..3a0395f8 100644 --- a/taskflow/task.py +++ b/taskflow/task.py @@ -35,6 +35,9 @@ REVERT_RESULT = 'result' # The cause of the flow failure/s REVERT_FLOW_FAILURES = 'flow_failures' +# Common events +EVENT_UPDATE_PROGRESS = 'update_progress' + @six.add_metaclass(abc.ABCMeta) class BaseTask(atom.Atom): @@ -46,7 +49,11 @@ class BaseTask(atom.Atom): same piece of work. """ - TASK_EVENTS = ('update_progress', ) + # Known events this task can have callbacks bound to (others that are not + # in this set/tuple will not be able to be bound); this should be updated + # and/or extended in subclasses as needed to enable or disable new or + # existing events... + TASK_EVENTS = (EVENT_UPDATE_PROGRESS,) def __init__(self, name, provides=None, inject=None): if name is None: @@ -139,83 +146,128 @@ class BaseTask(atom.Atom): if progress < 0.0: LOG.warn("Progress must be >= 0.0, clamping to lower bound") progress = 0.0 - self._trigger('update_progress', progress, **kwargs) + self.trigger(EVENT_UPDATE_PROGRESS, progress, **kwargs) - def _trigger(self, event, *args, **kwargs): - """Execute all handlers for the given event type.""" - for (handler, event_data) in self._events_listeners.get(event, []): + def trigger(self, event_name, *args, **kwargs): + """Execute all callbacks registered for the given event type. + + NOTE(harlowja): if a bound callback raises an exception it will be + logged (at a ``WARNING`` level) and the exception + will be dropped. + + :param event_name: event name to trigger + :param args: arbitrary positional arguments passed to the triggered + callbacks (if any are matched), these will be in addition + to any ``kwargs`` provided on binding (these are passed + as positional arguments to the callback). + :param kwargs: arbitrary keyword arguments passed to the triggered + callbacks (if any are matched), these will be in addition + to any ``kwargs`` provided on binding (these are passed + as keyword arguments to the callback). + """ + for (cb, event_data) in self._events_listeners.get(event_name, []): try: - handler(self, event_data, *args, **kwargs) + cb(self, event_data, *args, **kwargs) except Exception: - LOG.warn("Failed calling `%s` on event '%s'", - reflection.get_callable_name(handler), event, + LOG.warn("Failed calling callback `%s` on event '%s'", + reflection.get_callable_name(cb), event_name, exc_info=True) @contextlib.contextmanager - def autobind(self, event_name, handler_func, **kwargs): - """Binds & unbinds a given event handler to the task. + def autobind(self, event_name, callback, **kwargs): + """Binds & unbinds a given callback to the task. This function binds and unbinds using the context manager protocol. When events are triggered on the task of the given event name this - handler will automatically be called with the provided keyword - arguments. + callback will automatically be called with the provided + keyword arguments as the first argument (further arguments may be + provided by the entity triggering the event). + + The arguments are interpreted as for :func:`bind() `. """ bound = False - if handler_func is not None: + if callback is not None: try: - self.bind(event_name, handler_func, **kwargs) + self.bind(event_name, callback, **kwargs) bound = True except ValueError: - LOG.warn("Failed binding functor `%s` as a receiver of" - " event '%s' notifications emitted from task %s", - handler_func, event_name, self, exc_info=True) + LOG.warn("Failed binding callback `%s` as a receiver of" + " event '%s' notifications emitted from task '%s'", + reflection.get_callable_name(callback), event_name, + self, exc_info=True) try: yield self finally: if bound: - self.unbind(event_name, handler_func) + self.unbind(event_name, callback) - def bind(self, event, handler, **kwargs): - """Attach a handler to an event for the task. + def bind(self, event_name, callback, **kwargs): + """Attach a callback to be triggered on a task event. - :param event: event type - :param handler: callback to execute each time event is triggered + Callbacks should *not* be bound, modified, or removed after execution + has commenced (they may be adjusted after execution has finished). This + is primarily due to the need to preserve the callbacks that exist at + execution time for engines which run tasks remotely or out of + process (so that those engines can correctly proxy back transmitted + events). + + Callbacks should also be *quick* to execute so that the engine calling + them can continue execution in a timely manner (if long running + callbacks need to exist, consider creating a separate pool + queue + for those that the attached callbacks put long running operations into + for execution by other entities). + + :param event_name: event type name + :param callback: callable to execute each time event is triggered :param kwargs: optional named parameters that will be passed to the - event handler - :raises ValueError: if invalid event type passed + callable object as a dictionary to the callbacks + *second* positional parameter. + :raises ValueError: if invalid event type, or callback is passed """ - if event not in self.TASK_EVENTS: + if event_name not in self.TASK_EVENTS: raise ValueError("Unknown task event '%s', can only bind" - " to events %s" % (event, self.TASK_EVENTS)) - if not six.callable(handler): - raise ValueError("Event handler callback must be callable") - self._events_listeners[event].append((handler, kwargs)) + " to events %s" % (event_name, self.TASK_EVENTS)) + if callback is not None: + if not six.callable(callback): + raise ValueError("Event handler callback must be callable") + self._events_listeners[event_name].append((callback, kwargs)) - def unbind(self, event, handler=None): - """Remove a previously-attached event handler from the task. + def unbind(self, event_name, callback=None): + """Remove a previously-attached event callback from the task. - If a handler function not passed, then this will unbind all event - handlers for the provided event. If multiple of the same handlers are - bound, then the first match is removed (and only the first match). + If a callback is not passed, then this will unbind *all* event + callbacks for the provided event. If multiple of the same callbacks + are bound, then the first match is removed (and only the first match). - :param event: event type - :param handler: handler previously bound + :param event_name: event type + :param callback: callback previously bound :rtype: boolean :return: whether anything was removed """ removed_any = False - if not handler: - removed_any = self._events_listeners.pop(event, removed_any) + if not callback: + removed_any = self._events_listeners.pop(event_name, removed_any) else: - event_listeners = self._events_listeners.get(event, []) - for i, (handler2, _event_data) in enumerate(event_listeners): - if reflection.is_same_callback(handler, handler2): + event_listeners = self._events_listeners.get(event_name, []) + for i, (cb, _event_data) in enumerate(event_listeners): + if reflection.is_same_callback(cb, callback): + # NOTE(harlowja): its safe to do this as long as we stop + # iterating after we do the removal, otherwise its not + # safe (since this could have resized the list). event_listeners.pop(i) removed_any = True break return bool(removed_any) + def listeners_iter(self): + """Return an iterator over the mapping of event => callbacks bound.""" + for event_name in list(six.iterkeys(self._events_listeners)): + # Use get() just incase it was removed while iterating... + event_listeners = self._events_listeners.get(event_name, []) + if event_listeners: + yield (event_name, event_listeners[:]) + class Task(BaseTask): """Base class for user-defined tasks (derive from it at will!). diff --git a/taskflow/tests/unit/test_task.py b/taskflow/tests/unit/test_task.py index 8c9c7eff..3a963963 100644 --- a/taskflow/tests/unit/test_task.py +++ b/taskflow/tests/unit/test_task.py @@ -201,9 +201,9 @@ class TaskTest(test.TestCase): def progress_callback(task, event_data, progress): result.append(progress) - task = ProgressTask() - with task.autobind('update_progress', progress_callback): - task.execute(values) + a_task = ProgressTask() + with a_task.autobind(task.EVENT_UPDATE_PROGRESS, progress_callback): + a_task.execute(values) self.assertEqual(result, values) @mock.patch.object(task.LOG, 'warn') @@ -213,9 +213,9 @@ class TaskTest(test.TestCase): def progress_callback(task, event_data, progress): result.append(progress) - task = ProgressTask() - with task.autobind('update_progress', progress_callback): - task.execute([-1.0, -0.5, 0.0]) + a_task = ProgressTask() + with a_task.autobind(task.EVENT_UPDATE_PROGRESS, progress_callback): + a_task.execute([-1.0, -0.5, 0.0]) self.assertEqual(result, [0.0, 0.0, 0.0]) self.assertEqual(mocked_warn.call_count, 2) @@ -226,9 +226,9 @@ class TaskTest(test.TestCase): def progress_callback(task, event_data, progress): result.append(progress) - task = ProgressTask() - with task.autobind('update_progress', progress_callback): - task.execute([1.0, 1.5, 2.0]) + a_task = ProgressTask() + with a_task.autobind(task.EVENT_UPDATE_PROGRESS, progress_callback): + a_task.execute([1.0, 1.5, 2.0]) self.assertEqual(result, [1.0, 1.0, 1.0]) self.assertEqual(mocked_warn.call_count, 2) @@ -237,50 +237,40 @@ class TaskTest(test.TestCase): def progress_callback(*args, **kwargs): raise Exception('Woot!') - task = ProgressTask() - with task.autobind('update_progress', progress_callback): - task.execute([0.5]) + a_task = ProgressTask() + with a_task.autobind(task.EVENT_UPDATE_PROGRESS, progress_callback): + a_task.execute([0.5]) mocked_warn.assert_called_once_with( mock.ANY, reflection.get_callable_name(progress_callback), - 'update_progress', exc_info=mock.ANY) - - @mock.patch.object(task.LOG, 'warn') - def test_autobind_non_existent_event(self, mocked_warn): - event = 'test-event' - handler = lambda: None - task = MyTask() - with task.autobind(event, handler): - self.assertEqual(len(task._events_listeners), 0) - mocked_warn.assert_called_once_with( - mock.ANY, handler, event, task, exc_info=mock.ANY) + task.EVENT_UPDATE_PROGRESS, exc_info=mock.ANY) def test_autobind_handler_is_none(self): - task = MyTask() - with task.autobind('update_progress', None): - self.assertEqual(len(task._events_listeners), 0) + a_task = MyTask() + with a_task.autobind(task.EVENT_UPDATE_PROGRESS, None): + self.assertEqual(len(list(a_task.listeners_iter())), 0) def test_unbind_any_handler(self): - task = MyTask() - self.assertEqual(len(task._events_listeners), 0) - task.bind('update_progress', lambda: None) - self.assertEqual(len(task._events_listeners), 1) - self.assertTrue(task.unbind('update_progress')) - self.assertEqual(len(task._events_listeners), 0) + a_task = MyTask() + self.assertEqual(len(list(a_task.listeners_iter())), 0) + a_task.bind(task.EVENT_UPDATE_PROGRESS, lambda: None) + self.assertEqual(len(list(a_task.listeners_iter())), 1) + self.assertTrue(a_task.unbind(task.EVENT_UPDATE_PROGRESS)) + self.assertEqual(len(list(a_task.listeners_iter())), 0) def test_unbind_any_handler_empty_listeners(self): - task = MyTask() - self.assertEqual(len(task._events_listeners), 0) - self.assertFalse(task.unbind('update_progress')) - self.assertEqual(len(task._events_listeners), 0) + a_task = MyTask() + self.assertEqual(len(list(a_task.listeners_iter())), 0) + self.assertFalse(a_task.unbind(task.EVENT_UPDATE_PROGRESS)) + self.assertEqual(len(list(a_task.listeners_iter())), 0) def test_unbind_non_existent_listener(self): handler1 = lambda: None handler2 = lambda: None - task = MyTask() - task.bind('update_progress', handler1) - self.assertEqual(len(task._events_listeners), 1) - self.assertFalse(task.unbind('update_progress', handler2)) - self.assertEqual(len(task._events_listeners), 1) + a_task = MyTask() + a_task.bind(task.EVENT_UPDATE_PROGRESS, handler1) + self.assertEqual(len(list(a_task.listeners_iter())), 1) + self.assertFalse(a_task.unbind(task.EVENT_UPDATE_PROGRESS, handler2)) + self.assertEqual(len(list(a_task.listeners_iter())), 1) def test_bind_not_callable(self): task = MyTask() From af62f4c6747b561e6d0ccd0305021d8415f400d7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 12 Sep 2014 18:49:29 -0700 Subject: [PATCH 129/240] Ensure that failures can be pickled When a failure happens in a subprocess it needs to be pickleable (and not contain a traceback, since those can not be pickled) when being sent across the process boundary so that the receiving process can read it and unpickle the corresponding object. Part of blueprint process-executor Change-Id: I2f26faa4e02da6acf4f0840239d0b17143de8d76 --- taskflow/types/failure.py | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/taskflow/types/failure.py b/taskflow/types/failure.py index a0b3a17e..b9d7a399 100644 --- a/taskflow/types/failure.py +++ b/taskflow/types/failure.py @@ -34,6 +34,23 @@ def _copy_exc_info(exc_info): return (exc_type, copy.copy(exc_value), tb) +def _fill_iter(it, desired_len, filler=None): + """Iterates over a provided iterator up to the desired length. + + If the source iterator does not have enough values then the filler + value is yielded until the desired length is reached. + """ + count = 0 + for value in it: + if count >= desired_len: + return + yield value + count += 1 + while count < desired_len: + yield filler + count += 1 + + def _are_equal_exc_info_tuples(ei1, ei2): if ei1 == ei2: return True @@ -274,6 +291,30 @@ class Failure(object): for et in self._exc_type_names: yield et + def __getstate__(self): + dct = self.to_dict() + if self._exc_info: + # Avoids 'TypeError: can't pickle traceback objects' + dct['exc_info'] = self._exc_info[0:2] + return dct + + def __setstate__(self, dct): + self._exception_str = dct['exception_str'] + self._traceback_str = dct['traceback_str'] + self._exc_type_names = dct['exc_type_names'] + if 'exc_info' in dct: + # Tracebacks can't be serialized/deserialized, but since we + # provide a traceback string (and more) this should be + # acceptable... + # + # TODO(harlowja): in the future we could do something like + # what the twisted people have done, see for example + # twisted-13.0.0/twisted/python/failure.py#L89 for how they + # created a fake traceback object... + self._exc_info = tuple(_fill_iter(dct['exc_info'], 3)) + else: + self._exc_info = None + @classmethod def from_dict(cls, data): """Converts this from a dictionary to a object.""" From e978eca3635f4a92139e3c7c47a6b28586062cc0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 12 Sep 2014 18:49:29 -0700 Subject: [PATCH 130/240] Allow stopwatches to be restarted It is quite useful to allow a stopwatch to be reset or restarted in a single operation, instead of having to handle this logic outside of the stopwatch object, which is error-prone and unnecessary to require users to reinvent. Part of blueprint process-executor Change-Id: Ie0304a4f1d5f6bf0460e9a134762d1403da56375 --- taskflow/types/timing.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/taskflow/types/timing.py b/taskflow/types/timing.py index decada4a..ab9a4c48 100644 --- a/taskflow/types/timing.py +++ b/taskflow/types/timing.py @@ -73,6 +73,12 @@ class StopWatch(object): self._state = self._STARTED return self + def restart(self): + if self._state == self._STARTED: + self.stop() + self.start() + return self + def elapsed(self): if self._state == self._STOPPED: return max(0.0, float(timeutils.delta_seconds(self._started_at, From dc4262e58715a170f94cb5680f8b295115b64c92 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 12 Sep 2014 18:49:29 -0700 Subject: [PATCH 131/240] Have tasks be able to provide copy() methods When a engine needs to copy a task and possibly adjust its listeners to execute it elsewhere it needs to be able to clone that object and have the clone retain different properties than the initial object; so in order to support this at a top-level we require a new copy() method which a task can override (or it can just use the default implementation if it chooses to). Part of blueprint process-executor Change-Id: Ib29a0afdc01973eb94d41af18a9b04601cd2f152 --- taskflow/task.py | 19 +++++++++++++++++++ taskflow/tests/unit/test_task.py | 23 +++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/taskflow/task.py b/taskflow/task.py index 3a0395f8..c34a13bf 100644 --- a/taskflow/task.py +++ b/taskflow/task.py @@ -18,6 +18,7 @@ import abc import collections import contextlib +import copy import logging import six @@ -133,6 +134,24 @@ class BaseTask(atom.Atom): This works the same as :meth:`.post_execute`, but for the revert phase. """ + def copy(self, retain_listeners=True): + """Clone/copy this task. + + :param retain_listeners: retain the attached listeners when cloning, + when false the listeners will be emptied, when + true the listeners will be copied and retained + + :rtype: task + :return: the copied task + """ + c = copy.copy(self) + c._events_listeners = c._events_listeners.copy() + c._events_listeners.clear() + if retain_listeners: + for event_name, listeners in six.iteritems(self._events_listeners): + c._events_listeners[event_name] = listeners[:] + return c + def update_progress(self, progress, **kwargs): """Update task progress and notify all registered listeners. diff --git a/taskflow/tests/unit/test_task.py b/taskflow/tests/unit/test_task.py index 3a963963..b80c0c6a 100644 --- a/taskflow/tests/unit/test_task.py +++ b/taskflow/tests/unit/test_task.py @@ -276,6 +276,29 @@ class TaskTest(test.TestCase): task = MyTask() self.assertRaises(ValueError, task.bind, 'update_progress', 2) + def test_copy_no_listeners(self): + handler1 = lambda: None + a_task = MyTask() + a_task.bind(task.EVENT_UPDATE_PROGRESS, handler1) + b_task = a_task.copy(retain_listeners=False) + self.assertEqual(len(list(a_task.listeners_iter())), 1) + self.assertEqual(len(list(b_task.listeners_iter())), 0) + + def test_copy_listeners(self): + handler1 = lambda: None + handler2 = lambda: None + a_task = MyTask() + a_task.bind(task.EVENT_UPDATE_PROGRESS, handler1) + b_task = a_task.copy() + self.assertEqual(len(list(b_task.listeners_iter())), 1) + self.assertTrue(a_task.unbind(task.EVENT_UPDATE_PROGRESS)) + self.assertEqual(len(list(a_task.listeners_iter())), 0) + self.assertEqual(len(list(b_task.listeners_iter())), 1) + b_task.bind(task.EVENT_UPDATE_PROGRESS, handler2) + listeners = dict(list(b_task.listeners_iter())) + self.assertEqual(len(listeners[task.EVENT_UPDATE_PROGRESS]), 2) + self.assertEqual(len(list(a_task.listeners_iter())), 0) + class FunctorTaskTest(test.TestCase): From 5f0b514a1432c731cb9d7067d2699b24f4025dd1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 12 Sep 2014 18:49:29 -0700 Subject: [PATCH 132/240] Stop returning atoms from execute/revert methods It is not needed to return the atom that was executed from the futures result() method, since we can just as easily set an attribute on the future and reference it from there when using it later. This is also required for a process based executor since it is not typically possible to send back a raw task object (and is not desireable to require this); even if it was possible the task would be pickled and unpickled multiple times so when this happens it can not be guaranteed to even be the same object (in fact it is not). Part of blueprint process-executor Change-Id: I4a05ea5dcdef97218312e3a88ed4a1dfdf1b1edf --- .../engines/action_engine/actions/retry.py | 6 ++-- taskflow/engines/action_engine/executor.py | 29 ++++++++++++------- taskflow/engines/action_engine/runner.py | 3 +- taskflow/engines/worker_based/endpoint.py | 8 ++--- taskflow/engines/worker_based/protocol.py | 3 +- .../tests/unit/worker_based/test_pipeline.py | 17 +++++++---- .../tests/unit/worker_based/test_protocol.py | 3 +- 7 files changed, 43 insertions(+), 26 deletions(-) diff --git a/taskflow/engines/action_engine/actions/retry.py b/taskflow/engines/action_engine/actions/retry.py index 5afd2751..3262a79f 100644 --- a/taskflow/engines/action_engine/actions/retry.py +++ b/taskflow/engines/action_engine/actions/retry.py @@ -76,7 +76,7 @@ class RetryAction(object): result = retry.execute(**kwargs) except Exception: result = failure.Failure() - return (retry, ex.EXECUTED, result) + return (ex.EXECUTED, result) def _on_done_callback(fut): result = fut.result()[-1] @@ -89,6 +89,7 @@ class RetryAction(object): fut = self._executor.submit(_execute_retry, self._get_retry_args(retry)) fut.add_done_callback(_on_done_callback) + fut.atom = retry return fut def revert(self, retry): @@ -98,7 +99,7 @@ class RetryAction(object): result = retry.revert(**kwargs) except Exception: result = failure.Failure() - return (retry, ex.REVERTED, result) + return (ex.REVERTED, result) def _on_done_callback(fut): result = fut.result()[-1] @@ -115,6 +116,7 @@ class RetryAction(object): self._get_retry_args(retry, addons=arg_addons)) fut.add_done_callback(_on_done_callback) + fut.atom = retry return fut def on_failure(self, retry, atom, last_failure): diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index 4c5c091a..9224adb1 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -40,7 +40,7 @@ def _execute_task(task, arguments, progress_callback): result = failure.Failure() finally: task.post_execute() - return (task, EXECUTED, result) + return (EXECUTED, result) def _revert_task(task, arguments, result, failures, progress_callback): @@ -57,7 +57,7 @@ def _revert_task(task, arguments, result, failures, progress_callback): result = failure.Failure() finally: task.post_revert() - return (task, REVERTED, result) + return (REVERTED, result) @six.add_metaclass(abc.ABCMeta) @@ -98,13 +98,17 @@ class SerialTaskExecutor(TaskExecutorBase): self._executor = futures.SynchronousExecutor() def execute_task(self, task, task_uuid, arguments, progress_callback=None): - return self._executor.submit(_execute_task, task, arguments, - progress_callback) + fut = self._executor.submit(_execute_task, task, arguments, + progress_callback) + fut.atom = task + return fut def revert_task(self, task, task_uuid, arguments, result, failures, progress_callback=None): - return self._executor.submit(_revert_task, task, arguments, result, - failures, progress_callback) + fut = self._executor.submit(_revert_task, task, arguments, result, + failures, progress_callback) + fut.atom = task + return fut def wait_for_any(self, fs, timeout=None): return async_utils.wait_for_any(fs, timeout) @@ -123,14 +127,17 @@ class ParallelTaskExecutor(TaskExecutorBase): self._create_executor = executor is None def execute_task(self, task, task_uuid, arguments, progress_callback=None): - return self._executor.submit( - _execute_task, task, arguments, progress_callback) + fut = self._executor.submit(_execute_task, task, + arguments, progress_callback) + fut.atom = task + return fut def revert_task(self, task, task_uuid, arguments, result, failures, progress_callback=None): - return self._executor.submit( - _revert_task, task, - arguments, result, failures, progress_callback) + fut = self._executor.submit(_revert_task, task, arguments, + result, failures, progress_callback) + fut.atom = task + return fut def wait_for_any(self, fs, timeout=None): return async_utils.wait_for_any(fs, timeout) diff --git a/taskflow/engines/action_engine/runner.py b/taskflow/engines/action_engine/runner.py index 79ebc657..502bad18 100644 --- a/taskflow/engines/action_engine/runner.py +++ b/taskflow/engines/action_engine/runner.py @@ -129,8 +129,9 @@ class _MachineBuilder(object): next_nodes = set() while memory.done: fut = memory.done.pop() + node = fut.atom try: - node, event, result = fut.result() + event, result = fut.result() retain = self._completer.complete(node, event, result) if retain and isinstance(result, failure.Failure): memory.failures.append(result) diff --git a/taskflow/engines/worker_based/endpoint.py b/taskflow/engines/worker_based/endpoint.py index 3a16266d..276e93c6 100644 --- a/taskflow/engines/worker_based/endpoint.py +++ b/taskflow/engines/worker_based/endpoint.py @@ -40,11 +40,11 @@ class Endpoint(object): return self._task_cls(name=name) def execute(self, task_name, **kwargs): - task, event, result = self._executor.execute_task( - self._get_task(task_name), **kwargs).result() + task = self._get_task(task_name) + event, result = self._executor.execute_task(task, **kwargs).result() return result def revert(self, task_name, **kwargs): - task, event, result = self._executor.revert_task( - self._get_task(task_name), **kwargs).result() + task = self._get_task(task_name) + event, result = self._executor.revert_task(task, **kwargs).result() return result diff --git a/taskflow/engines/worker_based/protocol.py b/taskflow/engines/worker_based/protocol.py index 3cd7e178..3fd87216 100644 --- a/taskflow/engines/worker_based/protocol.py +++ b/taskflow/engines/worker_based/protocol.py @@ -234,6 +234,7 @@ class Request(Message): self._lock = threading.Lock() self._created_on = timeutils.utcnow() self.result = futures.Future() + self.result.atom = task @property def uuid(self): @@ -290,7 +291,7 @@ class Request(Message): return request def set_result(self, result): - self.result.set_result((self._task, self._event, result)) + self.result.set_result((self._event, result)) def on_progress(self, event_data, progress): self._progress_callback(self._task, event_data, progress) diff --git a/taskflow/tests/unit/worker_based/test_pipeline.py b/taskflow/tests/unit/worker_based/test_pipeline.py index ed3e2662..2822a852 100644 --- a/taskflow/tests/unit/worker_based/test_pipeline.py +++ b/taskflow/tests/unit/worker_based/test_pipeline.py @@ -16,6 +16,7 @@ from concurrent import futures +from taskflow.engines.action_engine import executor as base_executor from taskflow.engines.worker_based import endpoint from taskflow.engines.worker_based import executor as worker_executor from taskflow.engines.worker_based import server as worker_server @@ -73,13 +74,14 @@ class TestPipeline(test.TestCase): self.assertEqual(0, executor.wait_for_workers(timeout=WAIT_TIMEOUT)) t = test_utils.TaskOneReturn() - f = executor.execute_task(t, uuidutils.generate_uuid(), {}) + progress_callback = lambda *args, **kwargs: None + f = executor.execute_task(t, uuidutils.generate_uuid(), {}, + progress_callback=progress_callback) executor.wait_for_any([f]) - t2, _action, result = f.result() - + event, result = f.result() self.assertEqual(1, result) - self.assertEqual(t, t2) + self.assertEqual(base_executor.EXECUTED, event) def test_execution_failure_pipeline(self): task_classes = [ @@ -88,9 +90,12 @@ class TestPipeline(test.TestCase): executor, server = self._start_components(task_classes) t = test_utils.TaskWithFailure() - f = executor.execute_task(t, uuidutils.generate_uuid(), {}) + progress_callback = lambda *args, **kwargs: None + f = executor.execute_task(t, uuidutils.generate_uuid(), {}, + progress_callback=progress_callback) executor.wait_for_any([f]) - _t2, _action, result = f.result() + action, result = f.result() self.assertIsInstance(result, failure.Failure) self.assertEqual(RuntimeError, result.check(RuntimeError)) + self.assertEqual(base_executor.EXECUTED, action) diff --git a/taskflow/tests/unit/worker_based/test_protocol.py b/taskflow/tests/unit/worker_based/test_protocol.py index 4c9c4b77..2f5fd927 100644 --- a/taskflow/tests/unit/worker_based/test_protocol.py +++ b/taskflow/tests/unit/worker_based/test_protocol.py @@ -17,6 +17,7 @@ from concurrent import futures from oslo.utils import timeutils +from taskflow.engines.action_engine import executor from taskflow.engines.worker_based import protocol as pr from taskflow import exceptions as excp from taskflow.openstack.common import uuidutils @@ -182,7 +183,7 @@ class TestProtocol(test.TestCase): request = self.request() request.set_result(111) result = request.result.result() - self.assertEqual(result, (self.task, 'executed', 111)) + self.assertEqual(result, (executor.EXECUTED, 111)) def test_on_progress(self): progress_callback = mock.MagicMock(name='progress_callback') From cd664bdd3b83e544feb3a80c75e4037f5da72d39 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 7 Dec 2014 00:02:30 -0800 Subject: [PATCH 133/240] Correctly identify stack level in ``_extract_engine`` Add on the needed frame evaluation logic to be able to correctly locate where users frames start and ours end so that we can correctly report the stack level in the deprecation messages (which the warning module will use to show the location of the users code that is using the deprecated usage). Change-Id: Iae5919fb0f79ffbf9e1784723dc8da1cefdb9f27 --- taskflow/engines/helpers.py | 87 +++++++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 28 deletions(-) diff --git a/taskflow/engines/helpers.py b/taskflow/engines/helpers.py index 5877ee41..9ffff0dc 100644 --- a/taskflow/engines/helpers.py +++ b/taskflow/engines/helpers.py @@ -15,6 +15,8 @@ # under the License. import contextlib +import itertools +import traceback from oslo.utils import importutils import six @@ -34,39 +36,68 @@ ENGINES_NAMESPACE = 'taskflow.engines' # The default entrypoint engine type looked for when it is not provided. ENGINE_DEFAULT = 'default' +# TODO(harlowja): only used during the deprecation cycle, remove it once +# ``_extract_engine_compat`` is also gone... +_FILE_NAMES = [__file__] +if six.PY2: + # Due to a bug in py2.x the __file__ may point to the pyc file & since + # we are using the traceback module and that module only shows py files + # we have to do a slight adjustment to ensure we match correctly... + # + # This is addressed in https://www.python.org/dev/peps/pep-3147/#file + if __file__.endswith("pyc"): + _FILE_NAMES.append(__file__[0:-1]) +_FILE_NAMES = tuple(_FILE_NAMES) + -@deprecation.renamed_kwarg('engine_conf', 'engine', - version="0.6", removal_version="?", - # This is set to none since this function is called - # from 2 other functions in this module, both of - # which have different stack levels, possibly we - # can fix this in the future... - stacklevel=None) def _extract_engine(**kwargs): """Extracts the engine kind and any associated options.""" - options = {} - kind = kwargs.pop('engine', None) - engine_conf = kwargs.pop('engine_conf', None) - if engine_conf is not None: - if isinstance(engine_conf, six.string_types): - kind = engine_conf + + def _compat_extract(**kwargs): + options = {} + kind = kwargs.pop('engine', None) + engine_conf = kwargs.pop('engine_conf', None) + if engine_conf is not None: + if isinstance(engine_conf, six.string_types): + kind = engine_conf + else: + options.update(engine_conf) + kind = options.pop('engine', None) + if not kind: + kind = ENGINE_DEFAULT + # See if it's a URI and if so, extract any further options... + try: + uri = misc.parse_uri(kind) + except (TypeError, ValueError): + pass else: - options.update(engine_conf) - kind = options.pop('engine', None) - if not kind: - kind = ENGINE_DEFAULT - # See if it's a URI and if so, extract any further options... - try: - uri = misc.parse_uri(kind) - except (TypeError, ValueError): - pass + kind = uri.scheme + options = misc.merge_uri(uri, options.copy()) + # Merge in any leftover **kwargs into the options, this makes it so + # that the provided **kwargs override any URI or engine_conf specific + # options. + options.update(kwargs) + return (kind, options) + + engine_conf = kwargs.get('engine_conf', None) + if engine_conf is not None: + # Figure out where our code ends and the calling code begins (this is + # needed since this code is called from two functions in this module, + # which means the stack level will vary by one depending on that). + finder = itertools.takewhile(lambda frame: frame[0] in _FILE_NAMES, + reversed(traceback.extract_stack())) + stacklevel = sum(1 for _frame in finder) + decorator = deprecation.renamed_kwarg('engine_conf', 'engine', + version="0.6", + removal_version="?", + # Three is added on since the + # decorator adds three of its own + # stack levels that we need to + # hop out of... + stacklevel=stacklevel + 3) + return decorator(_compat_extract)(**kwargs) else: - kind = uri.scheme - options = misc.merge_uri(uri, options.copy()) - # Merge in any leftover **kwargs into the options, this makes it so that - # the provided **kwargs override any URI or engine_conf specific options. - options.update(kwargs) - return (kind, options) + return _compat_extract(**kwargs) def _fetch_factory(factory_name): From c4d327907d1f9147847c949b794292fc53aa262a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 7 Dec 2014 09:39:12 -0800 Subject: [PATCH 134/240] Allow all deprecation helpers to take a stacklevel To make the deprecation helpers more uniform in what arguments/keyword arguments they take allow them all to take a stacklevel keyword argument (which by default is set to a good value that works). This allows those users of this helper to override the stacklevel in a uniform manner (if they so need to). Change-Id: Icbfc115aa2aea4a5c01b99dcfa2a5c4e4785c458 --- taskflow/utils/deprecation.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/taskflow/utils/deprecation.py b/taskflow/utils/deprecation.py index db9c3790..a108676a 100644 --- a/taskflow/utils/deprecation.py +++ b/taskflow/utils/deprecation.py @@ -161,7 +161,8 @@ def renamed_kwarg(old_name, new_name, message=None, def _moved_decorator(kind, new_attribute_name, message=None, - version=None, removal_version=None): + version=None, removal_version=None, + stacklevel=3): """Decorates a method/property that was moved to another location.""" def decorator(f): @@ -184,7 +185,7 @@ def _moved_decorator(kind, new_attribute_name, message=None, out_message = _generate_moved_message( prefix, message=message, version=version, removal_version=removal_version) - deprecation(out_message, stacklevel=3) + deprecation(out_message, stacklevel=stacklevel) return f(self, *args, **kwargs) return wrapper @@ -193,15 +194,16 @@ def _moved_decorator(kind, new_attribute_name, message=None, def moved_property(new_attribute_name, message=None, - version=None, removal_version=None): + version=None, removal_version=None, stacklevel=3): """Decorates a *instance* property that was moved to another location.""" return _moved_decorator('Property', new_attribute_name, message=message, - version=version, removal_version=removal_version) + version=version, removal_version=removal_version, + stacklevel=stacklevel) def moved_class(new_class, old_class_name, old_module_name, message=None, - version=None, removal_version=None): + version=None, removal_version=None, stacklevel=3): """Deprecates a class that was moved to another location. This will emit warnings when the old locations class is initialized, @@ -213,4 +215,4 @@ def moved_class(new_class, old_class_name, old_module_name, message=None, out_message = _generate_moved_message(prefix, message=message, version=version, removal_version=removal_version) - return MovedClassProxy(new_class, out_message, stacklevel=3) + return MovedClassProxy(new_class, out_message, stacklevel=stacklevel) From 50b866cceba83eaf6133d21abda239f3d0ede069 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 7 Dec 2014 10:50:41 -0800 Subject: [PATCH 135/240] Use an appropriate ``extract_traceback`` limit When looking for where this modules ``run/load`` ends and the user code begins we only need to search back at most three levels to find this information; so to avoid getting many more frames (especially when they won't be used) then we need to just apply a sensible limit on the levels we get. Change-Id: I9186e8844c92697b7e3f850328b6b39afba0150d --- taskflow/engines/helpers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/taskflow/engines/helpers.py b/taskflow/engines/helpers.py index 9ffff0dc..61a729aa 100644 --- a/taskflow/engines/helpers.py +++ b/taskflow/engines/helpers.py @@ -84,8 +84,9 @@ def _extract_engine(**kwargs): # Figure out where our code ends and the calling code begins (this is # needed since this code is called from two functions in this module, # which means the stack level will vary by one depending on that). - finder = itertools.takewhile(lambda frame: frame[0] in _FILE_NAMES, - reversed(traceback.extract_stack())) + finder = itertools.takewhile( + lambda frame: frame[0] in _FILE_NAMES, + reversed(traceback.extract_stack(limit=3))) stacklevel = sum(1 for _frame in finder) decorator = deprecation.renamed_kwarg('engine_conf', 'engine', version="0.6", From 4eb0ca21b3cdf9b468a241bb18cfb38f3c7e9f83 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 7 Dec 2014 17:16:13 -0800 Subject: [PATCH 136/240] Use condition variables using 'with' Instead of doing [acquire, try, finally, release] just use the condition variable as a context manager to achieve the same effect with less code and with less verbosity. Change-Id: I0a3bb80a932a3dc6623ba2378afa0341e9e06e5a --- taskflow/engines/worker_based/executor.py | 10 ++-------- taskflow/jobs/backends/impl_zookeeper.py | 15 +++------------ taskflow/types/latch.py | 10 ++-------- 3 files changed, 7 insertions(+), 28 deletions(-) diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index 5982a463..092d4038 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -107,12 +107,9 @@ class WorkerTaskExecutor(executor.TaskExecutorBase): # Add worker info to the cache LOG.debug("Received that tasks %s can be processed by topic '%s'", tasks, topic) - self._workers_arrival.acquire() - try: + with self._workers_arrival: self._workers_cache[topic] = tasks self._workers_arrival.notify_all() - finally: - self._workers_arrival.release() # Publish waiting requests for request in self._requests_cache.get_waiting_requests(tasks): @@ -255,8 +252,7 @@ class WorkerTaskExecutor(executor.TaskExecutorBase): w = None if timeout is not None: w = tt.StopWatch(timeout).start() - self._workers_arrival.acquire() - try: + with self._workers_arrival: while len(self._workers_cache) < workers: if w is not None and w.expired(): return workers - len(self._workers_cache) @@ -265,8 +261,6 @@ class WorkerTaskExecutor(executor.TaskExecutorBase): timeout = w.leftover() self._workers_arrival.wait(timeout) return 0 - finally: - self._workers_arrival.release() def start(self): """Starts proxy thread and associated topic notification thread.""" diff --git a/taskflow/jobs/backends/impl_zookeeper.py b/taskflow/jobs/backends/impl_zookeeper.py index 05b519ec..ea485f8b 100644 --- a/taskflow/jobs/backends/impl_zookeeper.py +++ b/taskflow/jobs/backends/impl_zookeeper.py @@ -395,8 +395,7 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): LOG.warn("Internal error fetching job data from path: %s", path, exc_info=True) else: - self._job_cond.acquire() - try: + with self._job_cond: # Now we can offically check if someone already placed this # jobs information into the known job set (if it's already # existing then just leave it alone). @@ -409,8 +408,6 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): created_on=created_on) self._known_jobs[path] = job self._job_cond.notify_all() - finally: - self._job_cond.release() if job is not None: self._emit(jobboard.POSTED, details={'job': job}) @@ -488,12 +485,9 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): self._persistence, job_path, book=book, details=details, uuid=job_uuid) - self._job_cond.acquire() - try: + with self._job_cond: self._known_jobs[job_path] = job self._job_cond.notify_all() - finally: - self._job_cond.release() self._emit(jobboard.POSTED, details={'job': job}) return job @@ -634,8 +628,7 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): watch = None if timeout is not None: watch = tt.StopWatch(duration=float(timeout)).start() - self._job_cond.acquire() - try: + with self._job_cond: while True: if not self._known_jobs: if watch is not None and watch.expired(): @@ -656,8 +649,6 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): it._jobs.extend(self._fetch_jobs()) it._fetched = True return it - finally: - self._job_cond.release() @property def connected(self): diff --git a/taskflow/types/latch.py b/taskflow/types/latch.py index 0945a286..db6e56f3 100644 --- a/taskflow/types/latch.py +++ b/taskflow/types/latch.py @@ -41,13 +41,10 @@ class Latch(object): def countdown(self): """Decrements the internal counter due to an arrival.""" - self._cond.acquire() - try: + with self._cond: self._count -= 1 if self._count <= 0: self._cond.notify_all() - finally: - self._cond.release() def wait(self, timeout=None): """Waits until the latch is released. @@ -60,8 +57,7 @@ class Latch(object): w = None if timeout is not None: w = tt.StopWatch(timeout).start() - self._cond.acquire() - try: + with self._cond: while self._count > 0: if w is not None: if w.expired(): @@ -70,5 +66,3 @@ class Latch(object): timeout = w.leftover() self._cond.wait(timeout) return True - finally: - self._cond.release() From 2f78ecf1cc3ac98a2b586ab7bc95c6903eabf2f8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 8 Dec 2014 14:27:53 -0800 Subject: [PATCH 137/240] Add appropriate links into README.rst The oslo repositories are standardizing on a common set of links to include in README.rst (which the release note script will also use); so therefore we should add our own set to match those other projects. Change-Id: I58cbcd9b36fd0432b59e9eec3ad5ab54a1fede3a --- README.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index dea1f4ef..a19bc4d2 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,10 @@ A library to do [jobs, tasks, flows] in a highly available, easy to understand and declarative manner (and more!) to be used with OpenStack and other projects. -- More information can be found by referring to the `developer documentation`_. +* Free software: Apache license +* Documentation: http://docs.openstack.org/developer/taskflow +* Source: http://git.openstack.org/cgit/openstack/taskflow +* Bugs: http://bugs.launchpad.net/taskflow/ Join us ------- From a11336877fc4f8b99f02f73e3a67841340c8b23c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 8 Dec 2014 16:50:42 -0800 Subject: [PATCH 138/240] Have the sphinx copyright date be dynamic Instead of having the copyright date be statically encoded to '2013-2014' have it be dynamically picked up from the datetime module instead. Change-Id: Ie8a4ab02b7b2c254eb63a0b43763c9893f7fa083 --- doc/source/conf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index fcb07166..2fb1e7ec 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import datetime import os import sys @@ -36,7 +37,7 @@ exclude_patterns = ['_build'] # General information about the project. project = u'TaskFlow' -copyright = u'2013-2014, OpenStack Foundation' +copyright = u'%s, OpenStack Foundation' % datetime.date.today().year source_tree = 'http://git.openstack.org/cgit/openstack/taskflow/tree' # If true, '()' will be appended to :func: etc. cross-reference text. From 14431bc0769673fe5a70182189e2d4038a804cd8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 21 Sep 2014 10:37:35 -0700 Subject: [PATCH 139/240] Add and use a new simple helper logging module Add a new logging BLATHER level to easily allow its usage for messages that are below the normal DEBUG level such as compilation information and scope lookup info which can be very verbose in logs if always enabled. Change-Id: I828211403bd02bfd6777b10cdcfe58fb0637a52c --- taskflow/atom.py | 4 - taskflow/conductors/single_threaded.py | 3 +- .../engines/action_engine/actions/retry.py | 3 +- .../engines/action_engine/actions/task.py | 3 +- taskflow/engines/action_engine/compiler.py | 16 ++-- taskflow/engines/action_engine/runner.py | 3 +- taskflow/engines/action_engine/scopes.py | 16 +--- taskflow/engines/helpers.py | 3 + taskflow/engines/worker_based/dispatcher.py | 3 +- taskflow/engines/worker_based/executor.py | 2 +- taskflow/engines/worker_based/protocol.py | 2 +- taskflow/engines/worker_based/proxy.py | 2 +- taskflow/engines/worker_based/server.py | 2 +- taskflow/engines/worker_based/worker.py | 2 +- taskflow/jobs/backends/__init__.py | 2 +- taskflow/jobs/backends/impl_zookeeper.py | 2 +- taskflow/listeners/base.py | 2 +- taskflow/listeners/logging.py | 2 +- taskflow/listeners/timing.py | 2 +- taskflow/logging.py | 92 +++++++++++++++++++ taskflow/persistence/backends/__init__.py | 2 +- taskflow/persistence/backends/impl_dir.py | 2 +- taskflow/persistence/backends/impl_memory.py | 3 +- .../persistence/backends/impl_sqlalchemy.py | 2 +- .../persistence/backends/impl_zookeeper.py | 2 +- taskflow/persistence/logbook.py | 2 +- taskflow/retry.py | 3 - taskflow/storage.py | 21 ++--- taskflow/task.py | 2 +- taskflow/test.py | 2 + taskflow/utils/lock_utils.py | 2 +- taskflow/utils/misc.py | 2 - taskflow/utils/persistence_utils.py | 2 +- 33 files changed, 143 insertions(+), 70 deletions(-) create mode 100644 taskflow/logging.py diff --git a/taskflow/atom.py b/taskflow/atom.py index b3b7ea9c..2cd665ac 100644 --- a/taskflow/atom.py +++ b/taskflow/atom.py @@ -15,16 +15,12 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - import six from taskflow import exceptions from taskflow.utils import misc from taskflow.utils import reflection -LOG = logging.getLogger(__name__) - def _save_as_to_mapping(save_as): """Convert save_as to mapping name => index. diff --git a/taskflow/conductors/single_threaded.py b/taskflow/conductors/single_threaded.py index 173d2cec..e39f4c49 100644 --- a/taskflow/conductors/single_threaded.py +++ b/taskflow/conductors/single_threaded.py @@ -12,13 +12,12 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - import six from taskflow.conductors import base from taskflow import exceptions as excp from taskflow.listeners import logging as logging_listener +from taskflow import logging from taskflow.types import timing as tt from taskflow.utils import async_utils from taskflow.utils import lock_utils diff --git a/taskflow/engines/action_engine/actions/retry.py b/taskflow/engines/action_engine/actions/retry.py index 5afd2751..88a7c779 100644 --- a/taskflow/engines/action_engine/actions/retry.py +++ b/taskflow/engines/action_engine/actions/retry.py @@ -14,9 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - from taskflow.engines.action_engine import executor as ex +from taskflow import logging from taskflow import retry as retry_atom from taskflow import states from taskflow.types import failure diff --git a/taskflow/engines/action_engine/actions/task.py b/taskflow/engines/action_engine/actions/task.py index 6c520f16..b6bc6e77 100644 --- a/taskflow/engines/action_engine/actions/task.py +++ b/taskflow/engines/action_engine/actions/task.py @@ -14,8 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - +from taskflow import logging from taskflow import states from taskflow import task as task_atom from taskflow.types import failure diff --git a/taskflow/engines/action_engine/compiler.py b/taskflow/engines/action_engine/compiler.py index 47bc4a0b..38b48db6 100644 --- a/taskflow/engines/action_engine/compiler.py +++ b/taskflow/engines/action_engine/compiler.py @@ -14,11 +14,11 @@ # License for the specific language governing permissions and limitations # under the License. -import logging import threading from taskflow import exceptions as exc from taskflow import flow +from taskflow import logging from taskflow import retry from taskflow import task from taskflow.types import graph as gr @@ -190,18 +190,18 @@ class PatternCompiler(object): % (self._root, type(self._root))) self._history.clear() # NOTE(harlowja): this one can be expensive to calculate (especially - # the cycle detection), so only do it if we know debugging is enabled + # the cycle detection), so only do it if we know BLATHER is enabled # and not under all cases. - if LOG.isEnabledFor(logging.DEBUG): - LOG.debug("Translated '%s'", self._root) - LOG.debug("Graph:") + if LOG.isEnabledFor(logging.BLATHER): + LOG.blather("Translated '%s'", self._root) + LOG.blather("Graph:") for line in graph.pformat().splitlines(): # Indent it so that it's slightly offset from the above line. - LOG.debug(" %s", line) - LOG.debug("Hierarchy:") + LOG.blather(" %s", line) + LOG.blather("Hierarchy:") for line in node.pformat().splitlines(): # Indent it so that it's slightly offset from the above line. - LOG.debug(" %s", line) + LOG.blather(" %s", line) @lock_utils.locked def compile(self): diff --git a/taskflow/engines/action_engine/runner.py b/taskflow/engines/action_engine/runner.py index 79ebc657..e77aa918 100644 --- a/taskflow/engines/action_engine/runner.py +++ b/taskflow/engines/action_engine/runner.py @@ -14,8 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - +from taskflow import logging from taskflow import states as st from taskflow.types import failure from taskflow.types import fsm diff --git a/taskflow/engines/action_engine/scopes.py b/taskflow/engines/action_engine/scopes.py index f1dd49d1..6b7f9ffd 100644 --- a/taskflow/engines/action_engine/scopes.py +++ b/taskflow/engines/action_engine/scopes.py @@ -14,10 +14,9 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - from taskflow import atom as atom_type from taskflow import flow as flow_type +from taskflow import logging LOG = logging.getLogger(__name__) @@ -102,18 +101,13 @@ class ScopeWalker(object): visible.append(a) else: visible.append(a.name) - if LOG.isEnabledFor(logging.DEBUG): + if LOG.isEnabledFor(logging.BLATHER): if not self._names_only: visible_names = [a.name for a in visible] else: visible_names = visible - # TODO(harlowja): we should likely use a created TRACE level - # for this kind of *very* verbose information; otherwise the - # cinder and other folks are going to complain that there - # debug logs are full of not so useful information (it is - # useful to taskflow debugging...). - LOG.debug("Scope visible to '%s' (limited by parent '%s' index" - " < %s) is: %s", self._atom, parent.item.name, - last_idx, visible_names) + LOG.blather("Scope visible to '%s' (limited by parent '%s'" + " index < %s) is: %s", self._atom, + parent.item.name, last_idx, visible_names) yield visible last = parent diff --git a/taskflow/engines/helpers.py b/taskflow/engines/helpers.py index 9ffff0dc..5c2a6c32 100644 --- a/taskflow/engines/helpers.py +++ b/taskflow/engines/helpers.py @@ -23,12 +23,14 @@ import six import stevedore.driver from taskflow import exceptions as exc +from taskflow import logging from taskflow.persistence import backends as p_backends from taskflow.utils import deprecation from taskflow.utils import misc from taskflow.utils import persistence_utils as p_utils from taskflow.utils import reflection +LOG = logging.getLogger(__name__) # NOTE(imelnikov): this is the entrypoint namespace, not the module namespace. ENGINES_NAMESPACE = 'taskflow.engines' @@ -170,6 +172,7 @@ def load(flow, store=None, flow_detail=None, book=None, flow_detail = p_utils.create_flow_detail(flow, book=book, backend=backend) + LOG.debug('Looking for %r engine driver in %r', kind, namespace) try: mgr = stevedore.driver.DriverManager( namespace, kind, diff --git a/taskflow/engines/worker_based/dispatcher.py b/taskflow/engines/worker_based/dispatcher.py index f354ec4b..7ea8947c 100644 --- a/taskflow/engines/worker_based/dispatcher.py +++ b/taskflow/engines/worker_based/dispatcher.py @@ -14,12 +14,11 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - from kombu import exceptions as kombu_exc import six from taskflow import exceptions as excp +from taskflow import logging LOG = logging.getLogger(__name__) diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index 092d4038..3943a6e0 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -15,7 +15,6 @@ # under the License. import functools -import logging import threading from oslo.utils import timeutils @@ -25,6 +24,7 @@ from taskflow.engines.worker_based import cache from taskflow.engines.worker_based import protocol as pr from taskflow.engines.worker_based import proxy from taskflow import exceptions as exc +from taskflow import logging from taskflow.types import timing as tt from taskflow.utils import async_utils from taskflow.utils import misc diff --git a/taskflow/engines/worker_based/protocol.py b/taskflow/engines/worker_based/protocol.py index 3cd7e178..227b741f 100644 --- a/taskflow/engines/worker_based/protocol.py +++ b/taskflow/engines/worker_based/protocol.py @@ -15,7 +15,6 @@ # under the License. import abc -import logging import threading from concurrent import futures @@ -26,6 +25,7 @@ import six from taskflow.engines.action_engine import executor from taskflow import exceptions as excp +from taskflow import logging from taskflow.types import failure as ft from taskflow.types import timing as tt from taskflow.utils import lock_utils diff --git a/taskflow/engines/worker_based/proxy.py b/taskflow/engines/worker_based/proxy.py index 3f279a77..1c870595 100644 --- a/taskflow/engines/worker_based/proxy.py +++ b/taskflow/engines/worker_based/proxy.py @@ -15,13 +15,13 @@ # under the License. import collections -import logging import socket import kombu import six from taskflow.engines.worker_based import dispatcher +from taskflow import logging from taskflow.utils import threading_utils LOG = logging.getLogger(__name__) diff --git a/taskflow/engines/worker_based/server.py b/taskflow/engines/worker_based/server.py index db61edc6..0537dc09 100644 --- a/taskflow/engines/worker_based/server.py +++ b/taskflow/engines/worker_based/server.py @@ -15,12 +15,12 @@ # under the License. import functools -import logging import six from taskflow.engines.worker_based import protocol as pr from taskflow.engines.worker_based import proxy +from taskflow import logging from taskflow.types import failure as ft from taskflow.utils import misc diff --git a/taskflow/engines/worker_based/worker.py b/taskflow/engines/worker_based/worker.py index de55a8e2..18627e27 100644 --- a/taskflow/engines/worker_based/worker.py +++ b/taskflow/engines/worker_based/worker.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -import logging import os import platform import socket @@ -25,6 +24,7 @@ from concurrent import futures from taskflow.engines.worker_based import endpoint from taskflow.engines.worker_based import server +from taskflow import logging from taskflow import task as t_task from taskflow.utils import reflection from taskflow.utils import threading_utils as tu diff --git a/taskflow/jobs/backends/__init__.py b/taskflow/jobs/backends/__init__.py index 94afa6e5..e8bd6daf 100644 --- a/taskflow/jobs/backends/__init__.py +++ b/taskflow/jobs/backends/__init__.py @@ -15,12 +15,12 @@ # under the License. import contextlib -import logging import six from stevedore import driver from taskflow import exceptions as exc +from taskflow import logging from taskflow.utils import misc diff --git a/taskflow/jobs/backends/impl_zookeeper.py b/taskflow/jobs/backends/impl_zookeeper.py index ea485f8b..d6c131d6 100644 --- a/taskflow/jobs/backends/impl_zookeeper.py +++ b/taskflow/jobs/backends/impl_zookeeper.py @@ -17,7 +17,6 @@ import collections import contextlib import functools -import logging import threading from concurrent import futures @@ -31,6 +30,7 @@ import six from taskflow import exceptions as excp from taskflow.jobs import job as base_job from taskflow.jobs import jobboard +from taskflow import logging from taskflow.openstack.common import uuidutils from taskflow import states from taskflow.types import timing as tt diff --git a/taskflow/listeners/base.py b/taskflow/listeners/base.py index f69bb87e..47ff8d5b 100644 --- a/taskflow/listeners/base.py +++ b/taskflow/listeners/base.py @@ -17,11 +17,11 @@ from __future__ import absolute_import import abc -import logging from oslo.utils import excutils import six +from taskflow import logging from taskflow import states from taskflow.types import failure from taskflow.types import notifier diff --git a/taskflow/listeners/logging.py b/taskflow/listeners/logging.py index 51bf693c..d707e395 100644 --- a/taskflow/listeners/logging.py +++ b/taskflow/listeners/logging.py @@ -16,10 +16,10 @@ from __future__ import absolute_import -import logging import sys from taskflow.listeners import base +from taskflow import logging from taskflow import states from taskflow.types import failure diff --git a/taskflow/listeners/timing.py b/taskflow/listeners/timing.py index 4a08256e..f2080547 100644 --- a/taskflow/listeners/timing.py +++ b/taskflow/listeners/timing.py @@ -17,10 +17,10 @@ from __future__ import absolute_import import itertools -import logging from taskflow import exceptions as exc from taskflow.listeners import base +from taskflow import logging from taskflow import states from taskflow.types import timing as tt diff --git a/taskflow/logging.py b/taskflow/logging.py new file mode 100644 index 00000000..a735a9ae --- /dev/null +++ b/taskflow/logging.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import + +import logging +import sys + +_BASE = __name__.split(".", 1)[0] + +# Add a BLATHER level, this matches the multiprocessing utils.py module (and +# kazoo and others) that declares a similar level, this level is for +# information that is even lower level than regular DEBUG and gives out so +# much runtime information that it is only useful by low-level/certain users... +BLATHER = 5 + +# Copy over *select* attributes to make it easy to use this module. +CRITICAL = logging.CRITICAL +DEBUG = logging.DEBUG +ERROR = logging.ERROR +FATAL = logging.FATAL +NOTSET = logging.NOTSET +WARN = logging.WARN +WARNING = logging.WARNING + + +class _BlatherLoggerAdapter(logging.LoggerAdapter): + + def blather(self, msg, *args, **kwargs): + """Delegate a blather call to the underlying logger.""" + self.logger.log(BLATHER, msg, *args, **kwargs) + + def warn(self, msg, *args, **kwargs): + """Delegate a warning call to the underlying logger.""" + self.warning(msg, *args, **kwargs) + + +# TODO(harlowja): we should remove when we no longer have to support 2.6... +if sys.version_info[0:2] == (2, 6): + + class _FixedBlatherLoggerAdapter(_BlatherLoggerAdapter): + """Ensures isEnabledFor() exists on adapters that are created.""" + + def isEnabledFor(self, level): + return self.logger.isEnabledFor(level) + + _BlatherLoggerAdapter = _FixedBlatherLoggerAdapter + + # Taken from python2.7 (same in python3.4)... + class _NullHandler(logging.Handler): + """This handler does nothing. + + It's intended to be used to avoid the + "No handlers could be found for logger XXX" one-off warning. This is + important for library code, which may contain code to log events. If a + user of the library does not configure logging, the one-off warning + might be produced; to avoid this, the library developer simply needs + to instantiate a _NullHandler and add it to the top-level logger of the + library module or package. + """ + + def handle(self, record): + """Stub.""" + + def emit(self, record): + """Stub.""" + + def createLock(self): + self.lock = None + +else: + _NullHandler = logging.NullHandler + + +def getLogger(name=_BASE, extra=None): + logger = logging.getLogger(name) + if not logger.handlers: + logger.addHandler(_NullHandler()) + return _BlatherLoggerAdapter(logger, extra=extra) diff --git a/taskflow/persistence/backends/__init__.py b/taskflow/persistence/backends/__init__.py index 64b7cda1..cceb7e75 100644 --- a/taskflow/persistence/backends/__init__.py +++ b/taskflow/persistence/backends/__init__.py @@ -15,11 +15,11 @@ # under the License. import contextlib -import logging from stevedore import driver from taskflow import exceptions as exc +from taskflow import logging from taskflow.utils import misc diff --git a/taskflow/persistence/backends/impl_dir.py b/taskflow/persistence/backends/impl_dir.py index f469a1db..0a687473 100644 --- a/taskflow/persistence/backends/impl_dir.py +++ b/taskflow/persistence/backends/impl_dir.py @@ -16,7 +16,6 @@ # under the License. import errno -import logging import os import shutil @@ -24,6 +23,7 @@ from oslo.serialization import jsonutils import six from taskflow import exceptions as exc +from taskflow import logging from taskflow.persistence.backends import base from taskflow.persistence import logbook from taskflow.utils import lock_utils diff --git a/taskflow/persistence/backends/impl_memory.py b/taskflow/persistence/backends/impl_memory.py index f425987c..6c58718a 100644 --- a/taskflow/persistence/backends/impl_memory.py +++ b/taskflow/persistence/backends/impl_memory.py @@ -15,11 +15,10 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - import six from taskflow import exceptions as exc +from taskflow import logging from taskflow.persistence.backends import base from taskflow.persistence import logbook diff --git a/taskflow/persistence/backends/impl_sqlalchemy.py b/taskflow/persistence/backends/impl_sqlalchemy.py index 4b12b782..9241c7a4 100644 --- a/taskflow/persistence/backends/impl_sqlalchemy.py +++ b/taskflow/persistence/backends/impl_sqlalchemy.py @@ -22,7 +22,6 @@ from __future__ import absolute_import import contextlib import copy import functools -import logging import time from oslo.utils import strutils @@ -33,6 +32,7 @@ from sqlalchemy import orm as sa_orm from sqlalchemy import pool as sa_pool from taskflow import exceptions as exc +from taskflow import logging from taskflow.persistence.backends import base from taskflow.persistence.backends.sqlalchemy import migration from taskflow.persistence.backends.sqlalchemy import models diff --git a/taskflow/persistence/backends/impl_zookeeper.py b/taskflow/persistence/backends/impl_zookeeper.py index 3cacffec..ca801b43 100644 --- a/taskflow/persistence/backends/impl_zookeeper.py +++ b/taskflow/persistence/backends/impl_zookeeper.py @@ -15,13 +15,13 @@ # under the License. import contextlib -import logging from kazoo import exceptions as k_exc from kazoo.protocol import paths from oslo.serialization import jsonutils from taskflow import exceptions as exc +from taskflow import logging from taskflow.persistence.backends import base from taskflow.persistence import logbook from taskflow.utils import kazoo_utils as k_utils diff --git a/taskflow/persistence/logbook.py b/taskflow/persistence/logbook.py index f66f68ef..ea6de4d0 100644 --- a/taskflow/persistence/logbook.py +++ b/taskflow/persistence/logbook.py @@ -17,12 +17,12 @@ import abc import copy -import logging from oslo.utils import timeutils import six from taskflow import exceptions as exc +from taskflow import logging from taskflow.openstack.common import uuidutils from taskflow import states from taskflow.types import failure as ft diff --git a/taskflow/retry.py b/taskflow/retry.py index 6897e3b7..5e996be8 100644 --- a/taskflow/retry.py +++ b/taskflow/retry.py @@ -16,7 +16,6 @@ # under the License. import abc -import logging import six @@ -24,8 +23,6 @@ from taskflow import atom from taskflow import exceptions as exc from taskflow.utils import misc -LOG = logging.getLogger(__name__) - # Decision results. REVERT = "REVERT" REVERT_ALL = "REVERT_ALL" diff --git a/taskflow/storage.py b/taskflow/storage.py index c667509b..9a234ae5 100644 --- a/taskflow/storage.py +++ b/taskflow/storage.py @@ -16,11 +16,11 @@ import abc import contextlib -import logging import six from taskflow import exceptions +from taskflow import logging from taskflow.openstack.common import uuidutils from taskflow.persistence import logbook from taskflow import retry @@ -696,20 +696,17 @@ class Storage(object): injected_args = {} mapped_args = {} for (bound_name, name) in six.iteritems(args_mapping): - # TODO(harlowja): This logging information may be to verbose - # even for DEBUG mode, let's see if we can maybe in the future - # add a TRACE mode or something else if people complain... - if LOG.isEnabledFor(logging.DEBUG): + if LOG.isEnabledFor(logging.BLATHER): if atom_name: - LOG.debug("Looking for %r <= %r for atom named: %s", - bound_name, name, atom_name) + LOG.blather("Looking for %r <= %r for atom named: %s", + bound_name, name, atom_name) else: - LOG.debug("Looking for %r <= %r", bound_name, name) + LOG.blather("Looking for %r <= %r", bound_name, name) if name in injected_args: value = injected_args[name] mapped_args[bound_name] = value - LOG.debug("Matched %r <= %r to %r (from injected values)", - bound_name, name, value) + LOG.blather("Matched %r <= %r to %r (from injected" + " values)", bound_name, name, value) else: try: possible_providers = self._reverse_mapping[name] @@ -727,8 +724,8 @@ class Storage(object): % (bound_name, name, len(possible_providers))) provider, value = _item_from_first_of(providers, name) mapped_args[bound_name] = value - LOG.debug("Matched %r <= %r to %r (from %s)", - bound_name, name, value, provider) + LOG.blather("Matched %r <= %r to %r (from %s)", + bound_name, name, value, provider) return mapped_args def set_flow_state(self, state): diff --git a/taskflow/task.py b/taskflow/task.py index c34a13bf..278f2a22 100644 --- a/taskflow/task.py +++ b/taskflow/task.py @@ -19,11 +19,11 @@ import abc import collections import contextlib import copy -import logging import six from taskflow import atom +from taskflow import logging from taskflow.utils import reflection LOG = logging.getLogger(__name__) diff --git a/taskflow/test.py b/taskflow/test.py index c6a56a99..bf252229 100644 --- a/taskflow/test.py +++ b/taskflow/test.py @@ -14,6 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. +from __future__ import absolute_import + import collections import logging diff --git a/taskflow/utils/lock_utils.py b/taskflow/utils/lock_utils.py index 2d65395e..b61668b0 100644 --- a/taskflow/utils/lock_utils.py +++ b/taskflow/utils/lock_utils.py @@ -22,13 +22,13 @@ import collections import contextlib import errno -import logging import os import threading import time import six +from taskflow import logging from taskflow.utils import misc from taskflow.utils import threading_utils as tu diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 583cdd1a..88148bd7 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -19,7 +19,6 @@ import contextlib import datetime import errno import inspect -import logging import os import re import sys @@ -38,7 +37,6 @@ from taskflow.utils import deprecation from taskflow.utils import reflection -LOG = logging.getLogger(__name__) NUMERIC_TYPES = six.integer_types + (float,) # NOTE(imelnikov): regular expression to get scheme from URI, diff --git a/taskflow/utils/persistence_utils.py b/taskflow/utils/persistence_utils.py index dbcdac29..340f558a 100644 --- a/taskflow/utils/persistence_utils.py +++ b/taskflow/utils/persistence_utils.py @@ -15,10 +15,10 @@ # under the License. import contextlib -import logging from oslo.utils import timeutils +from taskflow import logging from taskflow.openstack.common import uuidutils from taskflow.persistence import logbook from taskflow.utils import misc From f333e1b2aacd8d5ed8efca76af6fc6d2e4535538 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 9 Dec 2014 00:14:38 -0800 Subject: [PATCH 140/240] Remove rtype from task clone() doc The sphinx docs don't seem to show this parameter correctly when building the html docs so instead of showing it just remove since the return type should be pretty obvious due to the surronding text. Change-Id: Id27c23d79d7fdbcd677ed959d9d817c796a0828c --- taskflow/task.py | 1 - 1 file changed, 1 deletion(-) diff --git a/taskflow/task.py b/taskflow/task.py index c34a13bf..a3d54e9a 100644 --- a/taskflow/task.py +++ b/taskflow/task.py @@ -141,7 +141,6 @@ class BaseTask(atom.Atom): when false the listeners will be emptied, when true the listeners will be copied and retained - :rtype: task :return: the copied task """ c = copy.copy(self) From b275c51a6fcf85d69af4e1ae7ee3d3918db8850b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 9 Dec 2014 15:52:14 -0800 Subject: [PATCH 141/240] Just assign a empty collection instead of copy/clear Instead of copying the '_events_listeners' then clear them to get a clean copy just set the copies '_events_listeners' to a new collection in the first place. Change-Id: I17d4f02241c90a49fc4334cfd1f2fffd03d58927 --- taskflow/task.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/taskflow/task.py b/taskflow/task.py index c34a13bf..26141775 100644 --- a/taskflow/task.py +++ b/taskflow/task.py @@ -145,8 +145,7 @@ class BaseTask(atom.Atom): :return: the copied task """ c = copy.copy(self) - c._events_listeners = c._events_listeners.copy() - c._events_listeners.clear() + c._events_listeners = collections.defaultdict(list) if retain_listeners: for event_name, listeners in six.iteritems(self._events_listeners): c._events_listeners[event_name] = listeners[:] From f1457a02aea3e32bffa3339f395597a38871b891 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 10 Dec 2014 18:58:42 -0800 Subject: [PATCH 142/240] Ensure message gets processed correctly Make sure that we call the local log() function and not the adapted loggers log function to ensure that we pick up the adapters potentially existing extra values. Change-Id: I6cb027bab5c32bc4a24be683aec88f397c0cd707 --- taskflow/logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskflow/logging.py b/taskflow/logging.py index a735a9ae..0ce457eb 100644 --- a/taskflow/logging.py +++ b/taskflow/logging.py @@ -41,7 +41,7 @@ class _BlatherLoggerAdapter(logging.LoggerAdapter): def blather(self, msg, *args, **kwargs): """Delegate a blather call to the underlying logger.""" - self.logger.log(BLATHER, msg, *args, **kwargs) + self.log(BLATHER, msg, *args, **kwargs) def warn(self, msg, *args, **kwargs): """Delegate a warning call to the underlying logger.""" From eaf49950383865438dd5aaef05de7d86977403f5 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 17 Nov 2014 16:28:49 -0800 Subject: [PATCH 143/240] Properly handle and skip empty intermediary flows Instead of linking nodes which have elements to predecessors which do not we should search backwards through the prior predecessors and link to one that does have nodes; this ensures that we do not create bad workflows when empty flows are injected. Fixes bug 1392650 Change-Id: Ic362ef3400f9c77e60ed07b0097e3427b999d1cd --- taskflow/engines/action_engine/compiler.py | 280 +++++++++++++++--- taskflow/exceptions.py | 4 + .../tests/unit/action_engine/test_compile.py | 101 +++++++ 3 files changed, 339 insertions(+), 46 deletions(-) diff --git a/taskflow/engines/action_engine/compiler.py b/taskflow/engines/action_engine/compiler.py index 38b48db6..a71b9d17 100644 --- a/taskflow/engines/action_engine/compiler.py +++ b/taskflow/engines/action_engine/compiler.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +import collections import threading from taskflow import exceptions as exc @@ -43,39 +44,241 @@ class Compilation(object): @property def execution_graph(self): + """The execution ordering of atoms (as a graph structure).""" return self._execution_graph @property def hierarchy(self): + """The hierachy of patterns (as a tree structure).""" return self._hierarchy +def _add_update_edges(graph, nodes_from, nodes_to, attr_dict=None): + """Adds/updates edges from nodes to other nodes in the specified graph. + + It will connect the 'nodes_from' to the 'nodes_to' if an edge currently + does *not* exist (if it does already exist then the edges attributes + are just updated instead). When an edge is created the provided edge + attributes dictionary will be applied to the new edge between these two + nodes. + """ + # NOTE(harlowja): give each edge its own attr copy so that if it's + # later modified that the same copy isn't modified... + for u in nodes_from: + for v in nodes_to: + if not graph.has_edge(u, v): + if attr_dict: + graph.add_edge(u, v, attr_dict=attr_dict.copy()) + else: + graph.add_edge(u, v) + else: + # Just update the attr_dict (if any). + if attr_dict: + graph.add_edge(u, v, attr_dict=attr_dict.copy()) + + +class Linker(object): + """Compiler helper that adds pattern(s) constraints onto a graph.""" + + @staticmethod + def _is_not_empty(graph): + # Returns true if the given graph is *not* empty... + return graph.number_of_nodes() > 0 + + @staticmethod + def _find_first_decomposed(node, priors, + decomposed_members, decomposed_filter): + # How this works; traverse backwards and find only the predecessor + # items that are actually connected to this entity, and avoid any + # linkage that is not directly connected. This is guaranteed to be + # valid since we always iter_links() over predecessors before + # successors in all currently known patterns; a queue is used here + # since it is possible for a node to have 2+ different predecessors so + # we must search back through all of them in a reverse BFS order... + # + # Returns the first decomposed graph of those nodes (including the + # passed in node) that passes the provided filter + # function (returns none if none match). + frontier = collections.deque([node]) + # NOTE(harowja): None is in this initial set since the first prior in + # the priors list has None as its predecessor (which we don't want to + # look for a decomposed member of). + visited = set([None]) + while frontier: + node = frontier.popleft() + if node in visited: + continue + node_graph = decomposed_members[node] + if decomposed_filter(node_graph): + return node_graph + visited.add(node) + # TODO(harlowja): optimize this more to avoid searching through + # things already searched... + for (u, v) in reversed(priors): + if node == v: + # Queue its predecessor to be searched in the future... + frontier.append(u) + else: + return None + + def apply_constraints(self, graph, flow, decomposed_members): + # This list is used to track the links that have been previously + # iterated over, so that when we are trying to find a entry to + # connect to that we iterate backwards through this list, finding + # connected nodes to the current target (lets call it v) and find + # the first (u_n, or u_n - 1, u_n - 2...) that was decomposed into + # a non-empty graph. We also retain all predecessors of v so that we + # can correctly locate u_n - 1 if u_n turns out to have decomposed into + # an empty graph (and so on). + priors = [] + # NOTE(harlowja): u, v are flows/tasks (also graph terminology since + # we are compiling things down into a flattened graph), the meaning + # of this link iteration via iter_links() is that u -> v (with the + # provided dictionary attributes, if any). + for (u, v, attr_dict) in flow.iter_links(): + if not priors: + priors.append((None, u)) + v_g = decomposed_members[v] + if not v_g.number_of_nodes(): + priors.append((u, v)) + continue + invariant = any(attr_dict.get(k) for k in _EDGE_INVARIANTS) + if not invariant: + # This is a symbol *only* dependency, connect + # corresponding providers and consumers to allow the consumer + # to be executed immediately after the provider finishes (this + # is an optimization for these types of dependencies...) + u_g = decomposed_members[u] + if not u_g.number_of_nodes(): + # This must always exist, but incase it somehow doesn't... + raise exc.CompilationFailure( + "Non-invariant link being created from '%s' ->" + " '%s' even though the target '%s' was found to be" + " decomposed into an empty graph" % (v, u, u)) + for provider in u_g: + for consumer in v_g: + reasons = provider.provides & consumer.requires + if reasons: + graph.add_edge(provider, consumer, reasons=reasons) + else: + # Connect nodes with no predecessors in v to nodes with no + # successors in the *first* non-empty predecessor of v (thus + # maintaining the edge dependency). + match = self._find_first_decomposed(u, priors, + decomposed_members, + self._is_not_empty) + if match is not None: + _add_update_edges(graph, + match.no_successors_iter(), + list(v_g.no_predecessors_iter()), + attr_dict=attr_dict) + priors.append((u, v)) + + class PatternCompiler(object): - """Compiles a pattern (or task) into a compilation unit.""" + """Compiles a pattern (or task) into a compilation unit. + + Let's dive into the basic idea for how this works: + + The compiler here is provided a 'root' object via its __init__ method, + this object could be a task, or a flow (one of the supported patterns), + the end-goal is to produce a :py:class:`.Compilation` object as the result + with the needed components. If this is not possible a + :py:class:`~.taskflow.exceptions.CompilationFailure` will be raised (or + in the case where a unknown type is being requested to compile + a ``TypeError`` will be raised). + + The complexity of this comes into play when the 'root' is a flow that + contains itself other nested flows (and so-on); to compile this object and + its contained objects into a graph that *preserves* the constraints the + pattern mandates we have to go through a recursive algorithm that creates + subgraphs for each nesting level, and then on the way back up through + the recursion (now with a decomposed mapping from contained patterns or + atoms to there corresponding subgraph) we have to then connect the + subgraphs (and the atom(s) there-in) that were decomposed for a pattern + correctly into a new graph (using a :py:class:`.Linker` object to ensure + the pattern mandated constraints are retained) and then return to the + caller (and they will do the same thing up until the root node, which by + that point one graph is created with all contained atoms in the + pattern/nested patterns mandated ordering). + + Also maintained in the :py:class:`.Compilation` object is a hierarchy of + the nesting of items (which is also built up during the above mentioned + recusion, via a much simpler algorithm); this is typically used later to + determine the prior atoms of a given atom when looking up values that can + be provided to that atom for execution (see the scopes.py file for how this + works). Note that although you *could* think that the graph itself could be + used for this, which in some ways it can (for limited usage) the hierarchy + retains the nested structure (which is useful for scoping analysis/lookup) + to be able to provide back a iterator that gives back the scopes visible + at each level (the graph does not have this information once flattened). + + Let's take an example: + + Given the pattern ``f(a(b, c), d)`` where ``f`` is a + :py:class:`~taskflow.patterns.linear_flow.Flow` with items ``a(b, c)`` + where ``a`` is a :py:class:`~taskflow.patterns.linear_flow.Flow` composed + of tasks ``(b, c)`` and task ``d``. + + The algorithm that will be performed (mirroring the above described logic) + will go through the following steps (the tree hierachy building is left + out as that is more obvious):: + + Compiling f + - Decomposing flow f with no parent (must be the root) + - Compiling a + - Decomposing flow a with parent f + - Compiling b + - Decomposing task b with parent a + - Decomposed b into: + Name: b + Nodes: 1 + - b + Edges: 0 + - Compiling c + - Decomposing task c with parent a + - Decomposed c into: + Name: c + Nodes: 1 + - c + Edges: 0 + - Relinking decomposed b -> decomposed c + - Decomposed a into: + Name: a + Nodes: 2 + - b + - c + Edges: 1 + b -> c ({'invariant': True}) + - Compiling d + - Decomposing task d with parent f + - Decomposed d into: + Name: d + Nodes: 1 + - d + Edges: 0 + - Relinking decomposed a -> decomposed d + - Decomposed f into: + Name: f + Nodes: 3 + - c + - b + - d + Edges: 2 + c -> d ({'invariant': True}) + b -> c ({'invariant': True}) + """ def __init__(self, root, freeze=True): self._root = root self._history = set() + self._linker = Linker() self._freeze = freeze self._lock = threading.Lock() self._compilation = None - def _add_new_edges(self, graph, nodes_from, nodes_to, edge_attrs): - """Adds new edges from nodes to other nodes in the specified graph. - - It will connect the nodes_from to the nodes_to if an edge currently - does *not* exist. When an edge is created the provided edge attributes - will be applied to the new edge between these two nodes. - """ - nodes_to = list(nodes_to) - for u in nodes_from: - for v in nodes_to: - if not graph.has_edge(u, v): - # NOTE(harlowja): give each edge its own attr copy so that - # if it's later modified that the same copy isn't modified. - graph.add_edge(u, v, attr_dict=edge_attrs.copy()) - def _flatten(self, item, parent): + """Flattens a item (pattern, task) into a graph + tree node.""" functor = self._find_flattener(item, parent) self._pre_item_flatten(item) graph, node = functor(item, parent) @@ -106,7 +309,9 @@ class PatternCompiler(object): # All nodes that have no predecessors should depend on this retry. nodes_to = [n for n in graph.no_predecessors_iter() if n is not retry] - self._add_new_edges(graph, [retry], nodes_to, _RETRY_EDGE_DATA) + if nodes_to: + _add_update_edges(graph, [retry], nodes_to, + attr_dict=_RETRY_EDGE_DATA) # Add association for each node of graph that has no existing retry. for n in graph.nodes_iter(): @@ -122,43 +327,26 @@ class PatternCompiler(object): parent.add(node) return graph, node - def _flatten_flow(self, flow, parent): - """Flattens a flow.""" + def _decompose_flow(self, flow, parent): + """Decomposes a flow into a graph, tree node + decomposed subgraphs.""" graph = gr.DiGraph(name=flow.name) node = tr.Node(flow) if parent is not None: parent.add(node) if flow.retry is not None: node.add(tr.Node(flow.retry)) - - # Flatten all nodes into a single subgraph per item (and track origin - # item to its newly expanded graph). - subgraphs = {} + decomposed_members = {} for item in flow: - subgraph = self._flatten(item, node)[0] - subgraphs[item] = subgraph - graph = gr.merge_graphs([graph, subgraph]) - - # Reconnect all items edges to their corresponding subgraphs. - for (u, v, attrs) in flow.iter_links(): - u_g = subgraphs[u] - v_g = subgraphs[v] - if any(attrs.get(k) for k in _EDGE_INVARIANTS): - # Connect nodes with no predecessors in v to nodes with - # no successors in u (thus maintaining the edge dependency). - self._add_new_edges(graph, - u_g.no_successors_iter(), - v_g.no_predecessors_iter(), - edge_attrs=attrs) - else: - # This is symbol dependency edge, connect corresponding - # providers and consumers. - for provider in u_g: - for consumer in v_g: - reasons = provider.provides & consumer.requires - if reasons: - graph.add_edge(provider, consumer, reasons=reasons) + subgraph, subnode = self._flatten(item, node) + decomposed_members[item] = subgraph + if subgraph.number_of_nodes(): + graph = gr.merge_graphs([graph, subgraph]) + return graph, node, decomposed_members + def _flatten_flow(self, flow, parent): + """Flattens a flow.""" + graph, node, decomposed_members = self._decompose_flow(flow, parent) + self._linker.apply_constraints(graph, flow, decomposed_members) if flow.retry is not None: self._connect_retry(flow.retry, graph) return graph, node diff --git a/taskflow/exceptions.py b/taskflow/exceptions.py index 1c6bc9f1..c6632754 100644 --- a/taskflow/exceptions.py +++ b/taskflow/exceptions.py @@ -135,6 +135,10 @@ class MissingDependencies(DependencyFailure): self.missing_requirements = requirements +class CompilationFailure(TaskFlowException): + """Raised when some type of compilation issue is found.""" + + class IncompatibleVersion(TaskFlowException): """Raised when some type of version incompatibility is found.""" diff --git a/taskflow/tests/unit/action_engine/test_compile.py b/taskflow/tests/unit/action_engine/test_compile.py index 63b3c0b0..a290c50b 100644 --- a/taskflow/tests/unit/action_engine/test_compile.py +++ b/taskflow/tests/unit/action_engine/test_compile.py @@ -264,6 +264,107 @@ class PatternCompileTest(test.TestCase): self.assertItemsEqual([b], g.no_predecessors_iter()) self.assertItemsEqual([a, c], g.no_successors_iter()) + def test_empty_flow_in_linear_flow(self): + flow = lf.Flow('lf') + a = test_utils.ProvidesRequiresTask('a', provides=[], requires=[]) + b = test_utils.ProvidesRequiresTask('b', provides=[], requires=[]) + empty_flow = gf.Flow("empty") + flow.add(a, empty_flow, b) + + compilation = compiler.PatternCompiler(flow).compile() + g = compilation.execution_graph + self.assertItemsEqual(g.edges(data=True), [ + (a, b, {'invariant': True}), + ]) + + def test_many_empty_in_graph_flow(self): + flow = gf.Flow('root') + + a = test_utils.ProvidesRequiresTask('a', provides=[], requires=[]) + flow.add(a) + + b = lf.Flow('b') + b_0 = test_utils.ProvidesRequiresTask('b.0', provides=[], requires=[]) + b_3 = test_utils.ProvidesRequiresTask('b.3', provides=[], requires=[]) + b.add( + b_0, + lf.Flow('b.1'), lf.Flow('b.2'), + b_3, + ) + flow.add(b) + + c = lf.Flow('c') + c.add(lf.Flow('c.0'), lf.Flow('c.1'), lf.Flow('c.2')) + flow.add(c) + + d = test_utils.ProvidesRequiresTask('d', provides=[], requires=[]) + flow.add(d) + + flow.link(b, d) + flow.link(a, d) + flow.link(c, d) + + compilation = compiler.PatternCompiler(flow).compile() + g = compilation.execution_graph + self.assertTrue(g.has_edge(b_0, b_3)) + self.assertTrue(g.has_edge(b_3, d)) + self.assertEqual(4, len(g)) + + def test_empty_flow_in_nested_flow(self): + flow = lf.Flow('lf') + a = test_utils.ProvidesRequiresTask('a', provides=[], requires=[]) + b = test_utils.ProvidesRequiresTask('b', provides=[], requires=[]) + + flow2 = lf.Flow("lf-2") + c = test_utils.ProvidesRequiresTask('c', provides=[], requires=[]) + d = test_utils.ProvidesRequiresTask('d', provides=[], requires=[]) + empty_flow = gf.Flow("empty") + flow2.add(c, empty_flow, d) + flow.add(a, flow2, b) + + compilation = compiler.PatternCompiler(flow).compile() + g = compilation.execution_graph + + self.assertTrue(g.has_edge(a, c)) + self.assertTrue(g.has_edge(c, d)) + self.assertTrue(g.has_edge(d, b)) + + def test_empty_flow_in_graph_flow(self): + flow = lf.Flow('lf') + a = test_utils.ProvidesRequiresTask('a', provides=['a'], requires=[]) + b = test_utils.ProvidesRequiresTask('b', provides=[], requires=['a']) + empty_flow = lf.Flow("empty") + flow.add(a, empty_flow, b) + + compilation = compiler.PatternCompiler(flow).compile() + g = compilation.execution_graph + self.assertTrue(g.has_edge(a, b)) + + def test_empty_flow_in_graph_flow_empty_linkage(self): + flow = gf.Flow('lf') + a = test_utils.ProvidesRequiresTask('a', provides=[], requires=[]) + b = test_utils.ProvidesRequiresTask('b', provides=[], requires=[]) + empty_flow = lf.Flow("empty") + flow.add(a, empty_flow, b) + flow.link(empty_flow, b) + + compilation = compiler.PatternCompiler(flow).compile() + g = compilation.execution_graph + self.assertEqual(0, len(g.edges())) + + def test_empty_flow_in_graph_flow_linkage(self): + flow = gf.Flow('lf') + a = test_utils.ProvidesRequiresTask('a', provides=[], requires=[]) + b = test_utils.ProvidesRequiresTask('b', provides=[], requires=[]) + empty_flow = lf.Flow("empty") + flow.add(a, empty_flow, b) + flow.link(a, b) + + compilation = compiler.PatternCompiler(flow).compile() + g = compilation.execution_graph + self.assertEqual(1, len(g.edges())) + self.assertTrue(g.has_edge(a, b)) + def test_checks_for_dups(self): flo = gf.Flow("test").add( test_utils.DummyTask(name="a"), From fda6fde26270c58a0d893d74086122ee45bda95a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 6 Dec 2014 12:15:01 -0800 Subject: [PATCH 144/240] Use explict 'attr_dict' when adding provider->consumer edge Instead of using the less explicit **kwarg support when adding an edge between a explicit producer and consumer use the 'attr_dict' keyword argument instead and use the constant defined the the flow module as the key into that dictionary (this also ensure that the key will be adjusted automatically if that key value ever changes). Change-Id: Ieeae83b984b7797320997c0c4cb4289eb1a837ee --- taskflow/engines/action_engine/compiler.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/taskflow/engines/action_engine/compiler.py b/taskflow/engines/action_engine/compiler.py index a71b9d17..c55a1e7f 100644 --- a/taskflow/engines/action_engine/compiler.py +++ b/taskflow/engines/action_engine/compiler.py @@ -33,6 +33,7 @@ _RETRY_EDGE_DATA = { flow.LINK_RETRY: True, } _EDGE_INVARIANTS = (flow.LINK_INVARIANT, flow.LINK_MANUAL, flow.LINK_RETRY) +_EDGE_REASONS = flow.LINK_REASONS class Compilation(object): @@ -155,11 +156,15 @@ class Linker(object): "Non-invariant link being created from '%s' ->" " '%s' even though the target '%s' was found to be" " decomposed into an empty graph" % (v, u, u)) - for provider in u_g: - for consumer in v_g: - reasons = provider.provides & consumer.requires - if reasons: - graph.add_edge(provider, consumer, reasons=reasons) + for u in u_g.nodes_iter(): + for v in v_g.nodes_iter(): + depends_on = u.provides & v.requires + if depends_on: + _add_update_edges(graph, + [u], [v], + attr_dict={ + _EDGE_REASONS: depends_on, + }) else: # Connect nodes with no predecessors in v to nodes with no # successors in the *first* non-empty predecessor of v (thus From a908124c6e3233c7083d429e9cf76eaa30781772 Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Thu, 11 Dec 2014 07:20:45 +0000 Subject: [PATCH 145/240] Updated from global requirements Change-Id: Ie7adab6aeff344c91151b4df6a55e5adb9d8333a --- requirements-py2.txt | 2 +- requirements-py3.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-py2.txt b/requirements-py2.txt index 0827e7ed..e1420074 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -26,5 +26,5 @@ futures>=2.1.6 jsonschema>=2.0.0,<3.0.0 # For common utilities -oslo.utils>=1.0.0 # Apache-2.0 +oslo.utils>=1.1.0 # Apache-2.0 oslo.serialization>=1.0.0 # Apache-2.0 diff --git a/requirements-py3.txt b/requirements-py3.txt index 5f265dcc..d827a17b 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -20,5 +20,5 @@ stevedore>=1.1.0 # Apache-2.0 jsonschema>=2.0.0,<3.0.0 # For common utilities -oslo.utils>=1.0.0 # Apache-2.0 +oslo.utils>=1.1.0 # Apache-2.0 oslo.serialization>=1.0.0 # Apache-2.0 From 880f7d28b95d01d3de36e4b8ac61768cd4310a59 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 10 Dec 2014 23:17:23 -0800 Subject: [PATCH 146/240] Avoid popping while another entity is iterating To avoid 'dictionary changed size during iteration' style of errors ensure that we acquire the job lock/condition before popping so that we do not affect another entity that is currently iterating over the job dictionary. Change-Id: I353cb7289c84c06f9712391e84294bed513dca78 --- taskflow/jobs/backends/impl_zookeeper.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/taskflow/jobs/backends/impl_zookeeper.py b/taskflow/jobs/backends/impl_zookeeper.py index d6c131d6..db873011 100644 --- a/taskflow/jobs/backends/impl_zookeeper.py +++ b/taskflow/jobs/backends/impl_zookeeper.py @@ -367,9 +367,12 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): ensure_fresh=ensure_fresh) def _remove_job(self, path): - LOG.debug("Removing job that was at path: %s", path) - job = self._known_jobs.pop(path, None) + if path not in self._known_jobs: + return + with self._job_cond: + job = self._known_jobs.pop(path, None) if job is not None: + LOG.debug("Removed job that was at path '%s'", path) self._emit(jobboard.REMOVAL, details={'job': job}) def _process_child(self, path, request): @@ -425,10 +428,11 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): # exist in the children anymore) and accumulate all paths that we # need to trigger population of (without holding the job lock). investigate_paths = [] + removals = [] with self._job_cond: for path in six.iterkeys(self._known_jobs): if path not in child_paths: - self._remove_job(path) + removals.append(path) for path in child_paths: if path in self._bad_paths: continue @@ -439,6 +443,10 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): continue if path not in investigate_paths: investigate_paths.append(path) + if removals: + with self._job_cond: + for path in removals: + self._remove_job(path) for path in investigate_paths: # Fire off the request to populate this job. # From 79ff9e7ea7fbb5c9bd022dd341eb28b68738d074 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 10 Dec 2014 23:04:30 -0800 Subject: [PATCH 147/240] Remove the base postfix for engine abstract base class Remove the non-standard pattern of calling the engine base class EngineBase; since it's already obvious that its a base class by it existing in the base engine module. Change-Id: Ia146ec6541ee96aa6a78fa659267d2a69e3b9e97 --- taskflow/engines/action_engine/engine.py | 2 +- taskflow/engines/base.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index 5a0fc9fb..ffc3a80a 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -42,7 +42,7 @@ def _start_stop(executor): executor.stop() -class ActionEngine(base.EngineBase): +class ActionEngine(base.Engine): """Generic action-based engine. This engine compiles the flow (and any subflows) into a compilation unit diff --git a/taskflow/engines/base.py b/taskflow/engines/base.py index 2c409726..dd477a52 100644 --- a/taskflow/engines/base.py +++ b/taskflow/engines/base.py @@ -25,7 +25,7 @@ from taskflow.utils import misc @six.add_metaclass(abc.ABCMeta) -class EngineBase(object): +class Engine(object): """Base for all engines implementations. :ivar notifier: A notification object that will dispatch events that From 1c7d242033c2ebf34ab73c6ba32308b9bba30ffd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 11 Dec 2014 13:15:19 -0800 Subject: [PATCH 148/240] Avoid holding the lock while scanning for existing jobs Under python we can safely check if a string key is in a dictionary without having to hold a lock on that dictionary to avoid concurrent deletions from affecting this check, so we can use that to our benefit to avoid holding the lock when searching for which potential jobs to request that do not already exist in the currently known jobs. Change-Id: I26c3bed2800789720cfc907c8e7dcbb3bad36447 --- taskflow/jobs/backends/impl_zookeeper.py | 30 +++++++++++++----------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/taskflow/jobs/backends/impl_zookeeper.py b/taskflow/jobs/backends/impl_zookeeper.py index db873011..e25ac51d 100644 --- a/taskflow/jobs/backends/impl_zookeeper.py +++ b/taskflow/jobs/backends/impl_zookeeper.py @@ -428,24 +428,26 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): # exist in the children anymore) and accumulate all paths that we # need to trigger population of (without holding the job lock). investigate_paths = [] - removals = [] + pending_removals = [] with self._job_cond: for path in six.iterkeys(self._known_jobs): if path not in child_paths: - removals.append(path) - for path in child_paths: - if path in self._bad_paths: - continue - # This pre-check will not guarantee that we will not already - # have the job (if it's being populated elsewhere) but it will - # reduce the amount of duplicated requests in general. - if path in self._known_jobs: - continue - if path not in investigate_paths: - investigate_paths.append(path) - if removals: + pending_removals.append(path) + for path in child_paths: + if path in self._bad_paths: + continue + # This pre-check will *not* guarantee that we will not already + # have the job (if it's being populated elsewhere) but it will + # reduce the amount of duplicated requests in general; later when + # the job information has been populated we will ensure that we + # are not adding duplicates into the currently known jobs... + if path in self._known_jobs: + continue + if path not in investigate_paths: + investigate_paths.append(path) + if pending_removals: with self._job_cond: - for path in removals: + for path in pending_removals: self._remove_job(path) for path in investigate_paths: # Fire off the request to populate this job. From a440ec40a518f8ee25ba5a945847e5b4eefdb45c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 5 Dec 2014 14:28:21 -0800 Subject: [PATCH 149/240] Add a moved_inheritable_class deprecation helper For cases where inheritance is still needed we should provide a mechanism to provide this so that consuming users of the library can still refer to the old location for a period of time so that there derived code does not fail due to the movement of the class that occurred. Change-Id: I8cd5b707125264f6b595d5b3ac327cfe16e7923c --- taskflow/utils/deprecation.py | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/taskflow/utils/deprecation.py b/taskflow/utils/deprecation.py index a108676a..f70d7670 100644 --- a/taskflow/utils/deprecation.py +++ b/taskflow/utils/deprecation.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +import functools import warnings import six @@ -202,6 +203,45 @@ def moved_property(new_attribute_name, message=None, stacklevel=stacklevel) +def moved_inheritable_class(new_class, old_class_name, old_module_name, + message=None, version=None, removal_version=None): + """Deprecates a class that was moved to another location. + + NOTE(harlowja): this creates a new-old type that can be used for a + deprecation period that can be inherited from, the difference between this + and the ``moved_class`` deprecation function is that the proxy from that + function can not be inherited from (thus limiting its use for a more + particular usecase where inheritance is not needed). + + This will emit warnings when the old locations class is initialized, + telling where the new and improved location for the old class now is. + """ + old_name = ".".join((old_module_name, old_class_name)) + new_name = reflection.get_class_name(new_class) + prefix = _CLASS_MOVED_PREFIX_TPL % (old_name, new_name) + out_message = _generate_moved_message(prefix, + message=message, version=version, + removal_version=removal_version) + + def decorator(f): + + # Use the older functools until the following is available: + # + # https://bitbucket.org/gutworth/six/issue/105 + + @functools.wraps(f, assigned=("__name__", "__doc__")) + def wrapper(self, *args, **kwargs): + deprecation(out_message, stacklevel=3) + return f(self, *args, **kwargs) + + return wrapper + + old_class = type(old_class_name, (new_class,), {}) + old_class.__module__ = old_module_name + old_class.__init__ = decorator(old_class.__init__) + return old_class + + def moved_class(new_class, old_class_name, old_module_name, message=None, version=None, removal_version=None, stacklevel=3): """Deprecates a class that was moved to another location. From b4e4e214cb69a3f2b9d90c287dcc2dd521a7f425 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 5 Dec 2014 14:28:21 -0800 Subject: [PATCH 150/240] Remove usage of listener base postfix The base prefix added on to the listener classes is not very useful and is already understood that these are base classes by documentation or abc.ABCMeta usage so we don't need to specifically name these classes bases to also show that they are a base class (useless duplicate information). So in this change we deprecate the 'ListenerBase' and the 'LoggingBase' and rename these classes to more appropriate names. Change-Id: Iaeaeaf698c23d71720ef7b62c8781996829e192a --- taskflow/listeners/base.py | 60 +++++++++++++++++++++++----------- taskflow/listeners/claims.py | 2 +- taskflow/listeners/logging.py | 6 ++-- taskflow/listeners/printing.py | 4 +-- taskflow/listeners/timing.py | 2 +- 5 files changed, 48 insertions(+), 26 deletions(-) diff --git a/taskflow/listeners/base.py b/taskflow/listeners/base.py index 47ff8d5b..0101e8de 100644 --- a/taskflow/listeners/base.py +++ b/taskflow/listeners/base.py @@ -25,6 +25,7 @@ from taskflow import logging from taskflow import states from taskflow.types import failure from taskflow.types import notifier +from taskflow.utils import deprecation LOG = logging.getLogger(__name__) @@ -80,7 +81,7 @@ def _bulk_register(watch_states, notifier, cb, details_filter=None): return registered -class ListenerBase(object): +class Listener(object): """Base class for listeners. A listener can be attached to an engine to do various actions on flow and @@ -162,26 +163,33 @@ class ListenerBase(object): self._engine, exc_info=True) +# TODO(harlowja): remove in 0.7 or later... +ListenerBase = deprecation.moved_inheritable_class(Listener, + 'ListenerBase', __name__, + version="0.6", + removal_version="?") + + @six.add_metaclass(abc.ABCMeta) -class LoggingBase(ListenerBase): - """Abstract base class for logging listeners. +class DumpingListener(Listener): + """Abstract base class for dumping listeners. This provides a simple listener that can be attached to an engine which can - be derived from to log task and/or flow state transitions to some logging + be derived from to dump task and/or flow state transitions to some target backend. - To implement your own logging listener derive form this class and - override the ``_log`` method. + To implement your own dumping listener derive from this class and + override the ``_dump`` method. """ @abc.abstractmethod - def _log(self, message, *args, **kwargs): - """Logs the provided *templated* message to some output.""" + def _dump(self, message, *args, **kwargs): + """Dumps the provided *templated* message to some output.""" def _flow_receiver(self, state, details): - self._log("%s has moved flow '%s' (%s) into state '%s'", - self._engine, details['flow_name'], - details['flow_uuid'], state) + self._dump("%s has moved flow '%s' (%s) into state '%s'", + self._engine, details['flow_name'], + details['flow_uuid'], state) def _task_receiver(self, state, details): if state in FINISH_STATES: @@ -192,12 +200,26 @@ class LoggingBase(ListenerBase): if result.exc_info: exc_info = tuple(result.exc_info) was_failure = True - self._log("%s has moved task '%s' (%s) into state '%s'" - " with result '%s' (failure=%s)", - self._engine, details['task_name'], - details['task_uuid'], state, result, was_failure, - exc_info=exc_info) + self._dump("%s has moved task '%s' (%s) into state '%s'" + " with result '%s' (failure=%s)", + self._engine, details['task_name'], + details['task_uuid'], state, result, was_failure, + exc_info=exc_info) else: - self._log("%s has moved task '%s' (%s) into state '%s'", - self._engine, details['task_name'], - details['task_uuid'], state) + self._dump("%s has moved task '%s' (%s) into state '%s'", + self._engine, details['task_name'], + details['task_uuid'], state) + + +# TODO(harlowja): remove in 0.7 or later... +class LoggingBase(deprecation.moved_inheritable_class(DumpingListener, + 'LoggingBase', __name__, + version="0.6", + removal_version="?")): + + def _dump(self, message, *args, **kwargs): + self._log(message, *args, **kwargs) + + @abc.abstractmethod + def _log(self, message, *args, **kwargs): + """Logs the provided *templated* message to some output.""" diff --git a/taskflow/listeners/claims.py b/taskflow/listeners/claims.py index 3fbc15d1..7fa86474 100644 --- a/taskflow/listeners/claims.py +++ b/taskflow/listeners/claims.py @@ -27,7 +27,7 @@ from taskflow import states LOG = logging.getLogger(__name__) -class CheckingClaimListener(base.ListenerBase): +class CheckingClaimListener(base.Listener): """Listener that interacts [engine, job, jobboard]; ensures claim is valid. This listener (or a derivative) can be associated with an engines diff --git a/taskflow/listeners/logging.py b/taskflow/listeners/logging.py index d707e395..03055257 100644 --- a/taskflow/listeners/logging.py +++ b/taskflow/listeners/logging.py @@ -41,7 +41,7 @@ def _isEnabledFor(logger, level): return logger.isEnabledFor(level) -class LoggingListener(base.LoggingBase): +class LoggingListener(base.DumpingListener): """Listener that logs notifications it receives. It listens for task and flow notifications and writes those notifications @@ -65,11 +65,11 @@ class LoggingListener(base.LoggingBase): self._logger = log self._level = level - def _log(self, message, *args, **kwargs): + def _dump(self, message, *args, **kwargs): self._logger.log(self._level, message, *args, **kwargs) -class DynamicLoggingListener(base.ListenerBase): +class DynamicLoggingListener(base.Listener): """Listener that logs notifications it receives. It listens for task and flow notifications and writes those notifications diff --git a/taskflow/listeners/printing.py b/taskflow/listeners/printing.py index 719d2042..2a89b179 100644 --- a/taskflow/listeners/printing.py +++ b/taskflow/listeners/printing.py @@ -22,7 +22,7 @@ import traceback from taskflow.listeners import base -class PrintingListener(base.LoggingBase): +class PrintingListener(base.DumpingListener): """Writes the task and flow notifications messages to stdout or stderr.""" def __init__(self, engine, task_listen_for=base.DEFAULT_LISTEN_FOR, @@ -37,7 +37,7 @@ class PrintingListener(base.LoggingBase): else: self._file = sys.stdout - def _log(self, message, *args, **kwargs): + def _dump(self, message, *args, **kwargs): print(message % args, file=self._file) exc_info = kwargs.get('exc_info') if exc_info is not None: diff --git a/taskflow/listeners/timing.py b/taskflow/listeners/timing.py index f2080547..c0fab524 100644 --- a/taskflow/listeners/timing.py +++ b/taskflow/listeners/timing.py @@ -39,7 +39,7 @@ def _printer(message): print(message) -class TimingListener(base.ListenerBase): +class TimingListener(base.Listener): """Listener that captures task duration. It records how long a task took to execute (or fail) From f5060ff41ea3fd2f9282c00a5a63f110918568b0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 12 Dec 2014 22:00:51 -0800 Subject: [PATCH 151/240] Remove the base postfix from the internal task executor There is no need to have a postfix of 'base' in the task executor as it is obvious by having a metaclass of abc.ABCMeta and the fact that it has no methods that the task executor base is a base class and should be used as such without a 'base' postfix to also signify the same information. Change-Id: I1f5cbcd29f1453d68e774f9f9f733eb873efc7cb --- taskflow/engines/action_engine/executor.py | 8 ++++---- taskflow/engines/worker_based/executor.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index 9224adb1..f91a615b 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -61,7 +61,7 @@ def _revert_task(task, arguments, result, failures, progress_callback): @six.add_metaclass(abc.ABCMeta) -class TaskExecutorBase(object): +class TaskExecutor(object): """Executes and reverts tasks. This class takes task and its arguments and executes or reverts it. @@ -91,8 +91,8 @@ class TaskExecutorBase(object): pass -class SerialTaskExecutor(TaskExecutorBase): - """Execute task one after another.""" +class SerialTaskExecutor(TaskExecutor): + """Executes tasks one after another.""" def __init__(self): self._executor = futures.SynchronousExecutor() @@ -114,7 +114,7 @@ class SerialTaskExecutor(TaskExecutorBase): return async_utils.wait_for_any(fs, timeout) -class ParallelTaskExecutor(TaskExecutorBase): +class ParallelTaskExecutor(TaskExecutor): """Executes tasks in parallel. Submits tasks to an executor which should provide an interface similar diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index 3943a6e0..bce92ccc 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -69,7 +69,7 @@ class PeriodicWorker(object): self._timeout.reset() -class WorkerTaskExecutor(executor.TaskExecutorBase): +class WorkerTaskExecutor(executor.TaskExecutor): """Executes tasks on remote workers.""" def __init__(self, uuid, exchange, topics, From 624d966e641a2a9f3fb22cb14bca94cb4dc7fcfa Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 12 Dec 2014 22:09:43 -0800 Subject: [PATCH 152/240] Retain the existence of a 'EngineBase' until 0.7 or later For the recently renamed 'EngineBase' -> 'Engine' cleanup it would be nice to retain the 'EngineBase' old named class for a deprecation cycle. To enable this use the newly made deprecation function that allows for creating inheritable classes that emit deprecation warnings when constructed. Change-Id: Ia67c924bc5896fbdc59bea25bf08fd87954905d0 --- taskflow/engines/base.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/taskflow/engines/base.py b/taskflow/engines/base.py index dd477a52..a97cf3b7 100644 --- a/taskflow/engines/base.py +++ b/taskflow/engines/base.py @@ -113,3 +113,10 @@ class Engine(object): not currently be preempted) and move the engine into a suspend state which can then later be resumed from. """ + + +# TODO(harlowja): remove in 0.7 or later... +EngineBase = deprecation.moved_inheritable_class(Engine, + 'EngineBase', __name__, + version="0.6", + removal_version="?") From cdfd8ece61c3340a3de73005ea7b44d0383223fd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 12 Dec 2014 23:03:12 -0800 Subject: [PATCH 153/240] Use a tiny clamp helper to clamp the 'on_progress' value Add a misc.clamp function that will clamp a value to a given range (it can also call a callback if clamping occurs). Use it to clamp the progress value that was previously clamped with a set of customized logic that can now be replaced with a more generalized logic that can be shared. Change-Id: I8369dbb61f73a60932d9e15c8b4d06db249ea38e --- taskflow/task.py | 14 +++++++------- taskflow/tests/unit/test_utils.py | 29 +++++++++++++++++++++++++++++ taskflow/utils/misc.py | 16 ++++++++++++++++ 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/taskflow/task.py b/taskflow/task.py index 11b9972d..ae947f1c 100644 --- a/taskflow/task.py +++ b/taskflow/task.py @@ -24,6 +24,7 @@ import six from taskflow import atom from taskflow import logging +from taskflow.utils import misc from taskflow.utils import reflection LOG = logging.getLogger(__name__) @@ -157,13 +158,12 @@ class BaseTask(atom.Atom): :param kwargs: any keyword arguments that are tied to the specific progress value. """ - if progress > 1.0: - LOG.warn("Progress must be <= 1.0, clamping to upper bound") - progress = 1.0 - if progress < 0.0: - LOG.warn("Progress must be >= 0.0, clamping to lower bound") - progress = 0.0 - self.trigger(EVENT_UPDATE_PROGRESS, progress, **kwargs) + def on_clamped(): + LOG.warn("Progress value must be greater or equal to 0.0 or less" + " than or equal to 1.0 instead of being '%s'", progress) + cleaned_progress = misc.clamp(progress, 0.0, 1.0, + on_clamped=on_clamped) + self.trigger(EVENT_UPDATE_PROGRESS, cleaned_progress, **kwargs) def trigger(self, event_name, *args, **kwargs): """Execute all callbacks registered for the given event type. diff --git a/taskflow/tests/unit/test_utils.py b/taskflow/tests/unit/test_utils.py index 66f08c09..56e39199 100644 --- a/taskflow/tests/unit/test_utils.py +++ b/taskflow/tests/unit/test_utils.py @@ -443,3 +443,32 @@ class TestSequenceMinus(test.TestCase): def test_equal_items_not_continious(self): result = misc.sequence_minus([1, 2, 3, 1], [1, 3]) self.assertEqual(result, [2, 1]) + + +class TestClamping(test.TestCase): + def test_simple_clamp(self): + result = misc.clamp(1.0, 2.0, 3.0) + self.assertEqual(result, 2.0) + result = misc.clamp(4.0, 2.0, 3.0) + self.assertEqual(result, 3.0) + result = misc.clamp(3.0, 4.0, 4.0) + self.assertEqual(result, 4.0) + + def test_invalid_clamp(self): + self.assertRaises(ValueError, misc.clamp, 0.0, 2.0, 1.0) + + def test_clamped_callback(self): + calls = [] + + def on_clamped(): + calls.append(True) + + misc.clamp(-1, 0.0, 1.0, on_clamped=on_clamped) + self.assertEqual(1, len(calls)) + calls.pop() + + misc.clamp(0.0, 0.0, 1.0, on_clamped=on_clamped) + self.assertEqual(0, len(calls)) + + misc.clamp(2, 0.0, 1.0, on_clamped=on_clamped) + self.assertEqual(1, len(calls)) diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 88148bd7..34910064 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -100,6 +100,22 @@ def parse_uri(uri): query=split.query) +def clamp(value, minimum, maximum, on_clamped=None): + """Clamps a value to ensure its >= minimum and <= maximum.""" + if minimum > maximum: + raise ValueError("Provided minimum '%s' must be less than or equal to" + " the provided maximum '%s'" % (minimum, maximum)) + if value > maximum: + value = maximum + if on_clamped is not None: + on_clamped() + if value < minimum: + value = minimum + if on_clamped is not None: + on_clamped() + return value + + def binary_encode(text, encoding='utf-8'): """Converts a string of into a binary type using given encoding. From 1f4dd72e6e859f3820de5e68ece2d09c15a18b70 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 12 Dec 2014 15:57:53 -0800 Subject: [PATCH 154/240] Use the notifier type in the task class/module directly Instead of having code that is some what like the notifier code we already have, but is duplicated and is slightly different in the task class just move the code that was in the task class (and doing similar actions) to instead now use a notifier that is directly contained in the task base class for internal task triggering of internal task events. Breaking change: alters the capabilities of the task to process notifications itself, most actions now must go through the task notifier property and instead use that (update_progress still exists as a common utility method, since it's likely the most common type of notification that will be used). Removes the following methods from task base class (as they are no longer needed with a notifier attribute): - trigger (replaced with notifier.notify) - autobind (removed, not replaced, can be created by the user of taskflow in a simple manner, without requiring functionality in taskflow) - bind (replaced with notifier.register) - unbind (replaced with notifier.unregister) - listeners_iter (replaced with notifier.listeners_iter) Due to this change we can now also correctly proxy back events from remote tasks to the engine for correct proxying back to the original task. Fixes bug 1370766 Change-Id: Ic9dfef516d72e6e32e71dda30a1cb3522c9e0be6 --- doc/source/workers.rst | 87 +++++++-- .../engines/action_engine/actions/task.py | 55 ++++-- taskflow/engines/action_engine/executor.py | 54 ++++-- taskflow/engines/worker_based/endpoint.py | 8 +- taskflow/engines/worker_based/executor.py | 22 ++- taskflow/engines/worker_based/protocol.py | 36 ++-- taskflow/engines/worker_based/server.py | 68 ++++--- taskflow/task.py | 166 +++--------------- taskflow/tests/unit/test_notifier.py | 25 +++ taskflow/tests/unit/test_progress.py | 25 +-- taskflow/tests/unit/test_task.py | 116 ++++++------ .../tests/unit/worker_based/test_endpoint.py | 12 +- .../tests/unit/worker_based/test_executor.py | 20 ++- .../unit/worker_based/test_message_pump.py | 2 +- .../tests/unit/worker_based/test_protocol.py | 25 +-- .../tests/unit/worker_based/test_server.py | 53 +++--- taskflow/types/notifier.py | 82 ++++++++- 17 files changed, 492 insertions(+), 364 deletions(-) diff --git a/doc/source/workers.rst b/doc/source/workers.rst index 4bb5b362..373615ef 100644 --- a/doc/source/workers.rst +++ b/doc/source/workers.rst @@ -158,10 +158,11 @@ engine executor in the following manner: from dicts after receiving on both executor & worker sides (this translation is lossy since the traceback won't be fully retained). -Executor request format +Executor execute format ~~~~~~~~~~~~~~~~~~~~~~~ -* **task** - full task name to be performed +* **task_name** - full task name to be performed +* **task_cls** - full task class name to be performed * **action** - task action to be performed (e.g. execute, revert) * **arguments** - arguments the task action to be called with * **result** - task execution result (result or @@ -180,9 +181,14 @@ Additionally, the following parameters are added to the request message: { "action": "execute", "arguments": { - "joe_number": 444 + "x": 111 }, - "task": "tasks.CallJoe" + "task_cls": "taskflow.tests.utils.TaskOneArgOneReturn", + "task_name": "taskflow.tests.utils.TaskOneArgOneReturn", + "task_version": [ + 1, + 0 + ] } Worker response format @@ -193,7 +199,8 @@ When **running:** .. code:: json { - "status": "RUNNING" + "data": {}, + "state": "RUNNING" } When **progressing:** @@ -201,9 +208,11 @@ When **progressing:** .. code:: json { - "event_data": , - "progress": , - "state": "PROGRESS" + "details": { + "progress": 0.5 + }, + "event_type": "update_progress", + "state": "EVENT" } When **succeeded:** @@ -211,8 +220,9 @@ When **succeeded:** .. code:: json { - "event": , - "result": , + "data": { + "result": 666 + }, "state": "SUCCESS" } @@ -221,11 +231,64 @@ When **failed:** .. code:: json { - "event": , - "result": , + "data": { + "result": { + "exc_type_names": [ + "RuntimeError", + "StandardError", + "Exception" + ], + "exception_str": "Woot!", + "traceback_str": " File \"/homes/harlowja/dev/os/taskflow/taskflow/engines/action_engine/executor.py\", line 56, in _execute_task\n result = task.execute(**arguments)\n File \"/homes/harlowja/dev/os/taskflow/taskflow/tests/utils.py\", line 165, in execute\n raise RuntimeError('Woot!')\n", + "version": 1 + } + }, "state": "FAILURE" } +Executor revert format +~~~~~~~~~~~~~~~~~~~~~~ + +When **reverting:** + +.. code:: json + + { + "action": "revert", + "arguments": {}, + "failures": { + "taskflow.tests.utils.TaskWithFailure": { + "exc_type_names": [ + "RuntimeError", + "StandardError", + "Exception" + ], + "exception_str": "Woot!", + "traceback_str": " File \"/homes/harlowja/dev/os/taskflow/taskflow/engines/action_engine/executor.py\", line 56, in _execute_task\n result = task.execute(**arguments)\n File \"/homes/harlowja/dev/os/taskflow/taskflow/tests/utils.py\", line 165, in execute\n raise RuntimeError('Woot!')\n", + "version": 1 + } + }, + "result": [ + "failure", + { + "exc_type_names": [ + "RuntimeError", + "StandardError", + "Exception" + ], + "exception_str": "Woot!", + "traceback_str": " File \"/homes/harlowja/dev/os/taskflow/taskflow/engines/action_engine/executor.py\", line 56, in _execute_task\n result = task.execute(**arguments)\n File \"/homes/harlowja/dev/os/taskflow/taskflow/tests/utils.py\", line 165, in execute\n raise RuntimeError('Woot!')\n", + "version": 1 + } + ], + "task_cls": "taskflow.tests.utils.TaskWithFailure", + "task_name": "taskflow.tests.utils.TaskWithFailure", + "task_version": [ + 1, + 0 + ] + } + Usage ===== diff --git a/taskflow/engines/action_engine/actions/task.py b/taskflow/engines/action_engine/actions/task.py index b6bc6e77..ccf450b5 100644 --- a/taskflow/engines/action_engine/actions/task.py +++ b/taskflow/engines/action_engine/actions/task.py @@ -14,6 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. +import functools + from taskflow import logging from taskflow import states from taskflow import task as task_atom @@ -75,25 +77,37 @@ class TaskAction(object): if progress is not None: task.update_progress(progress) - def _on_update_progress(self, task, event_data, progress, **kwargs): + def _on_update_progress(self, task, event_type, details): """Should be called when task updates its progress.""" try: - self._storage.set_task_progress(task.name, progress, kwargs) - except Exception: - # Update progress callbacks should never fail, so capture and log - # the emitted exception instead of raising it. - LOG.exception("Failed setting task progress for %s to %0.3f", - task, progress) + progress = details.pop('progress') + except KeyError: + pass + else: + try: + self._storage.set_task_progress(task.name, progress, + details=details) + except Exception: + # Update progress callbacks should never fail, so capture and + # log the emitted exception instead of raising it. + LOG.exception("Failed setting task progress for %s to %0.3f", + task, progress) def schedule_execution(self, task): self.change_state(task, states.RUNNING, progress=0.0) scope_walker = self._walker_factory(task) - kwargs = self._storage.fetch_mapped_args(task.rebind, - atom_name=task.name, - scope_walker=scope_walker) + arguments = self._storage.fetch_mapped_args(task.rebind, + atom_name=task.name, + scope_walker=scope_walker) + if task.notifier.can_be_registered(task_atom.EVENT_UPDATE_PROGRESS): + progress_callback = functools.partial(self._on_update_progress, + task) + else: + progress_callback = None task_uuid = self._storage.get_atom_uuid(task.name) - return self._task_executor.execute_task(task, task_uuid, kwargs, - self._on_update_progress) + return self._task_executor.execute_task( + task, task_uuid, arguments, + progress_callback=progress_callback) def complete_execution(self, task, result): if isinstance(result, failure.Failure): @@ -105,15 +119,20 @@ class TaskAction(object): def schedule_reversion(self, task): self.change_state(task, states.REVERTING, progress=0.0) scope_walker = self._walker_factory(task) - kwargs = self._storage.fetch_mapped_args(task.rebind, - atom_name=task.name, - scope_walker=scope_walker) + arguments = self._storage.fetch_mapped_args(task.rebind, + atom_name=task.name, + scope_walker=scope_walker) task_uuid = self._storage.get_atom_uuid(task.name) task_result = self._storage.get(task.name) failures = self._storage.get_failures() - future = self._task_executor.revert_task(task, task_uuid, kwargs, - task_result, failures, - self._on_update_progress) + if task.notifier.can_be_registered(task_atom.EVENT_UPDATE_PROGRESS): + progress_callback = functools.partial(self._on_update_progress, + task) + else: + progress_callback = None + future = self._task_executor.revert_task( + task, task_uuid, arguments, task_result, failures, + progress_callback=progress_callback) return future def complete_reversion(self, task, rev_result): diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index f91a615b..afe37d1d 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -15,10 +15,11 @@ # under the License. import abc +import contextlib import six -from taskflow import task as _task +from taskflow import task as task_atom from taskflow.types import failure from taskflow.types import futures from taskflow.utils import async_utils @@ -29,8 +30,23 @@ EXECUTED = 'executed' REVERTED = 'reverted' -def _execute_task(task, arguments, progress_callback): - with task.autobind(_task.EVENT_UPDATE_PROGRESS, progress_callback): +@contextlib.contextmanager +def _autobind(task, progress_callback=None): + bound = False + if progress_callback is not None: + task.notifier.register(task_atom.EVENT_UPDATE_PROGRESS, + progress_callback) + bound = True + try: + yield + finally: + if bound: + task.notifier.deregister(task_atom.EVENT_UPDATE_PROGRESS, + progress_callback) + + +def _execute_task(task, arguments, progress_callback=None): + with _autobind(task, progress_callback=progress_callback): try: task.pre_execute() result = task.execute(**arguments) @@ -43,14 +59,14 @@ def _execute_task(task, arguments, progress_callback): return (EXECUTED, result) -def _revert_task(task, arguments, result, failures, progress_callback): - kwargs = arguments.copy() - kwargs[_task.REVERT_RESULT] = result - kwargs[_task.REVERT_FLOW_FAILURES] = failures - with task.autobind(_task.EVENT_UPDATE_PROGRESS, progress_callback): +def _revert_task(task, arguments, result, failures, progress_callback=None): + arguments = arguments.copy() + arguments[task_atom.REVERT_RESULT] = result + arguments[task_atom.REVERT_FLOW_FAILURES] = failures + with _autobind(task, progress_callback=progress_callback): try: task.pre_revert() - result = task.revert(**kwargs) + result = task.revert(**arguments) except Exception: # NOTE(imelnikov): wrap current exception with Failure # object and return it. @@ -98,15 +114,17 @@ class SerialTaskExecutor(TaskExecutor): self._executor = futures.SynchronousExecutor() def execute_task(self, task, task_uuid, arguments, progress_callback=None): - fut = self._executor.submit(_execute_task, task, arguments, - progress_callback) + fut = self._executor.submit(_execute_task, + task, arguments, + progress_callback=progress_callback) fut.atom = task return fut def revert_task(self, task, task_uuid, arguments, result, failures, progress_callback=None): - fut = self._executor.submit(_revert_task, task, arguments, result, - failures, progress_callback) + fut = self._executor.submit(_revert_task, + task, arguments, result, failures, + progress_callback=progress_callback) fut.atom = task return fut @@ -127,15 +145,17 @@ class ParallelTaskExecutor(TaskExecutor): self._create_executor = executor is None def execute_task(self, task, task_uuid, arguments, progress_callback=None): - fut = self._executor.submit(_execute_task, task, - arguments, progress_callback) + fut = self._executor.submit(_execute_task, + task, arguments, + progress_callback=progress_callback) fut.atom = task return fut def revert_task(self, task, task_uuid, arguments, result, failures, progress_callback=None): - fut = self._executor.submit(_revert_task, task, arguments, - result, failures, progress_callback) + fut = self._executor.submit(_revert_task, + task, arguments, result, failures, + progress_callback=progress_callback) fut.atom = task return fut diff --git a/taskflow/engines/worker_based/endpoint.py b/taskflow/engines/worker_based/endpoint.py index 276e93c6..58637e11 100644 --- a/taskflow/engines/worker_based/endpoint.py +++ b/taskflow/engines/worker_based/endpoint.py @@ -33,18 +33,16 @@ class Endpoint(object): def name(self): return self._task_cls_name - def _get_task(self, name=None): + def generate(self, name=None): # NOTE(skudriashev): Note that task is created here with the `name` # argument passed to its constructor. This will be a problem when # task's constructor requires any other arguments. return self._task_cls(name=name) - def execute(self, task_name, **kwargs): - task = self._get_task(task_name) + def execute(self, task, **kwargs): event, result = self._executor.execute_task(task, **kwargs).result() return result - def revert(self, task_name, **kwargs): - task = self._get_task(task_name) + def revert(self, task, **kwargs): event, result = self._executor.revert_task(task, **kwargs).result() return result diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index bce92ccc..cd068e81 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -25,6 +25,7 @@ from taskflow.engines.worker_based import protocol as pr from taskflow.engines.worker_based import proxy from taskflow import exceptions as exc from taskflow import logging +from taskflow import task as task_atom from taskflow.types import timing as tt from taskflow.utils import async_utils from taskflow.utils import misc @@ -132,8 +133,11 @@ class WorkerTaskExecutor(executor.TaskExecutor): response.state, request) if response.state == pr.RUNNING: request.transition_and_log_error(pr.RUNNING, logger=LOG) - elif response.state == pr.PROGRESS: - request.on_progress(**response.data) + elif response.state == pr.EVENT: + # Proxy the event + details to the task/request notifier... + event_type = response.data['event_type'] + details = response.data['details'] + request.notifier.notify(event_type, details) elif response.state in (pr.FAILURE, pr.SUCCESS): moved = request.transition_and_log_error(response.state, logger=LOG) @@ -181,8 +185,18 @@ class WorkerTaskExecutor(executor.TaskExecutor): progress_callback, **kwargs): """Submit task request to a worker.""" request = pr.Request(task, task_uuid, action, arguments, - progress_callback, self._transition_timeout, - **kwargs) + self._transition_timeout, **kwargs) + + # Register the callback, so that we can proxy the progress correctly. + if (progress_callback is not None and + request.notifier.can_be_registered( + task_atom.EVENT_UPDATE_PROGRESS)): + request.notifier.register(task_atom.EVENT_UPDATE_PROGRESS, + progress_callback) + cleaner = functools.partial(request.notifier.deregister, + task_atom.EVENT_UPDATE_PROGRESS, + progress_callback) + request.result.add_done_callback(lambda fut: cleaner()) # Get task's topic and publish request if topic was found. topic = self._workers_cache.get_topic_by_task(request.task_cls) diff --git a/taskflow/engines/worker_based/protocol.py b/taskflow/engines/worker_based/protocol.py index c2bb22d2..e2b40e60 100644 --- a/taskflow/engines/worker_based/protocol.py +++ b/taskflow/engines/worker_based/protocol.py @@ -38,14 +38,14 @@ PENDING = 'PENDING' RUNNING = 'RUNNING' SUCCESS = 'SUCCESS' FAILURE = 'FAILURE' -PROGRESS = 'PROGRESS' +EVENT = 'EVENT' # During these states the expiry is active (once out of these states the expiry # no longer matters, since we have no way of knowing how long a task will run # for). WAITING_STATES = (WAITING, PENDING) -_ALL_STATES = (WAITING, PENDING, RUNNING, SUCCESS, FAILURE, PROGRESS) +_ALL_STATES = (WAITING, PENDING, RUNNING, SUCCESS, FAILURE, EVENT) _STOP_TIMER_STATES = (RUNNING, SUCCESS, FAILURE) # Transitions that a request state can go through. @@ -219,22 +219,29 @@ class Request(Message): 'required': ['task_cls', 'task_name', 'task_version', 'action'], } - def __init__(self, task, uuid, action, arguments, progress_callback, - timeout, **kwargs): + def __init__(self, task, uuid, action, arguments, timeout, **kwargs): self._task = task self._task_cls = reflection.get_class_name(task) self._uuid = uuid self._action = action self._event = ACTION_TO_EVENT[action] self._arguments = arguments - self._progress_callback = progress_callback self._kwargs = kwargs self._watch = tt.StopWatch(duration=timeout).start() self._state = WAITING self._lock = threading.Lock() self._created_on = timeutils.utcnow() - self.result = futures.Future() - self.result.atom = task + self._result = futures.Future() + self._result.atom = task + self._notifier = task.notifier + + @property + def result(self): + return self._result + + @property + def notifier(self): + return self._notifier @property def uuid(self): @@ -293,9 +300,6 @@ class Request(Message): def set_result(self, result): self.result.set_result((self._event, result)) - def on_progress(self, event_data, progress): - self._progress_callback(self._task, event_data, progress) - def transition_and_log_error(self, new_state, logger=None): """Transitions *and* logs an error if that transitioning raises. @@ -362,7 +366,7 @@ class Response(Message): 'data': { "anyOf": [ { - "$ref": "#/definitions/progress", + "$ref": "#/definitions/event", }, { "$ref": "#/definitions/completion", @@ -376,17 +380,17 @@ class Response(Message): "required": ["state", 'data'], "additionalProperties": False, "definitions": { - "progress": { + "event": { "type": "object", "properties": { - 'progress': { - 'type': 'number', + 'event_type': { + 'type': 'string', }, - 'event_data': { + 'details': { 'type': 'object', }, }, - "required": ["progress", 'event_data'], + "required": ["event_type", 'details'], "additionalProperties": False, }, # Used when sending *only* request state changes (and no data is diff --git a/taskflow/engines/worker_based/server.py b/taskflow/engines/worker_based/server.py index 0537dc09..bb7a97d2 100644 --- a/taskflow/engines/worker_based/server.py +++ b/taskflow/engines/worker_based/server.py @@ -22,6 +22,7 @@ from taskflow.engines.worker_based import protocol as pr from taskflow.engines.worker_based import proxy from taskflow import logging from taskflow.types import failure as ft +from taskflow.types import notifier as nt from taskflow.utils import misc LOG = logging.getLogger(__name__) @@ -73,18 +74,23 @@ class Server(object): All `failure.Failure` objects that have been converted to dict on the remote side will now converted back to `failure.Failure` objects. """ - action_args = dict(arguments=arguments, task_name=task_name) + # These arguments will eventually be given to the task executor + # so they need to be in a format it will accept (and using keyword + # argument names that it accepts)... + arguments = { + 'arguments': arguments, + } if result is not None: data_type, data = result if data_type == 'failure': - action_args['result'] = ft.Failure.from_dict(data) + arguments['result'] = ft.Failure.from_dict(data) else: - action_args['result'] = data + arguments['result'] = data if failures is not None: - action_args['failures'] = {} + arguments['failures'] = {} for key, data in six.iteritems(failures): - action_args['failures'][key] = ft.Failure.from_dict(data) - return task_cls, action, action_args + arguments['failures'][key] = ft.Failure.from_dict(data) + return (task_cls, task_name, action, arguments) @staticmethod def _parse_message(message): @@ -122,14 +128,13 @@ class Server(object): exc_info=True) return published - def _on_update_progress(self, reply_to, task_uuid, task, event_data, - progress): - """Send task update progress notification.""" + def _on_event(self, reply_to, task_uuid, event_type, details): + """Send out a task event notification.""" # NOTE(harlowja): the executor that will trigger this using the # task notification/listener mechanism will handle logging if this # fails, so thats why capture is 'False' is used here. - self._reply(False, reply_to, task_uuid, pr.PROGRESS, - event_data=event_data, progress=progress) + self._reply(False, reply_to, task_uuid, pr.EVENT, + event_type=event_type, details=details) def _process_notify(self, notify, message): """Process notify message and reply back.""" @@ -165,18 +170,15 @@ class Server(object): message.delivery_tag, exc_info=True) return else: - # prepare task progress callback - progress_callback = functools.partial(self._on_update_progress, - reply_to, task_uuid) # prepare reply callback reply_callback = functools.partial(self._reply, True, reply_to, task_uuid) # parse request to get task name, action and action arguments try: - task_cls, action, action_args = self._parse_request(**request) - action_args.update(task_uuid=task_uuid, - progress_callback=progress_callback) + bundle = self._parse_request(**request) + task_cls, task_name, action, arguments = bundle + arguments['task_uuid'] = task_uuid except ValueError: with misc.capture_failure() as failure: LOG.warn("Failed to parse request contents from message %r", @@ -206,12 +208,36 @@ class Server(object): reply_callback(result=failure.to_dict()) return else: - if not reply_callback(state=pr.RUNNING): - return + try: + task = endpoint.generate(name=task_name) + except Exception: + with misc.capture_failure() as failure: + LOG.warn("The '%s' task '%s' generation for request" + " message %r failed", endpoint, action, + message.delivery_tag, exc_info=True) + reply_callback(result=failure.to_dict()) + return + else: + if not reply_callback(state=pr.RUNNING): + return - # perform task action + # associate *any* events this task emits with a proxy that will + # emit them back to the engine... for handling at the engine side + # of things... + if task.notifier.can_be_registered(nt.Notifier.ANY): + task.notifier.register(nt.Notifier.ANY, + functools.partial(self._on_event, + reply_to, task_uuid)) + elif isinstance(task.notifier, nt.RestrictedNotifier): + # only proxy the allowable events then... + for event_type in task.notifier.events_iter(): + task.notifier.register(event_type, + functools.partial(self._on_event, + reply_to, task_uuid)) + + # perform the task action try: - result = handler(**action_args) + result = handler(task, **arguments) except Exception: with misc.capture_failure() as failure: LOG.warn("The '%s' endpoint '%s' execution for request" diff --git a/taskflow/task.py b/taskflow/task.py index ae947f1c..fbee0296 100644 --- a/taskflow/task.py +++ b/taskflow/task.py @@ -16,14 +16,13 @@ # under the License. import abc -import collections -import contextlib import copy import six from taskflow import atom from taskflow import logging +from taskflow.types import notifier from taskflow.utils import misc from taskflow.utils import reflection @@ -51,18 +50,28 @@ class BaseTask(atom.Atom): same piece of work. """ - # Known events this task can have callbacks bound to (others that are not - # in this set/tuple will not be able to be bound); this should be updated - # and/or extended in subclasses as needed to enable or disable new or - # existing events... + # Known internal events this task can have callbacks bound to (others that + # are not in this set/tuple will not be able to be bound); this should be + # updated and/or extended in subclasses as needed to enable or disable new + # or existing internal events... TASK_EVENTS = (EVENT_UPDATE_PROGRESS,) def __init__(self, name, provides=None, inject=None): if name is None: name = reflection.get_class_name(self) super(BaseTask, self).__init__(name, provides, inject=inject) - # Map of events => lists of callbacks to invoke on task events. - self._events_listeners = collections.defaultdict(list) + self._notifier = notifier.RestrictedNotifier(self.TASK_EVENTS) + + @property + def notifier(self): + """Internal notification dispatcher/registry. + + A notification object that will dispatch events that occur related + to *internal* notifications that the task internally emits to + listeners (for example for progress status updates, telling others + that a task has reached 50% completion...). + """ + return self._notifier def pre_execute(self): """Code to be run prior to executing the task. @@ -138,152 +147,31 @@ class BaseTask(atom.Atom): def copy(self, retain_listeners=True): """Clone/copy this task. - :param retain_listeners: retain the attached listeners when cloning, - when false the listeners will be emptied, when - true the listeners will be copied and retained + :param retain_listeners: retain the attached notification listeners + when cloning, when false the listeners will + be emptied, when true the listeners will be + copied and retained :return: the copied task """ c = copy.copy(self) - c._events_listeners = collections.defaultdict(list) - if retain_listeners: - for event_name, listeners in six.iteritems(self._events_listeners): - c._events_listeners[event_name] = listeners[:] + c._notifier = self._notifier.copy() + if not retain_listeners: + c._notifier.reset() return c - def update_progress(self, progress, **kwargs): + def update_progress(self, progress): """Update task progress and notify all registered listeners. :param progress: task progress float value between 0.0 and 1.0 - :param kwargs: any keyword arguments that are tied to the specific - progress value. """ def on_clamped(): LOG.warn("Progress value must be greater or equal to 0.0 or less" " than or equal to 1.0 instead of being '%s'", progress) cleaned_progress = misc.clamp(progress, 0.0, 1.0, on_clamped=on_clamped) - self.trigger(EVENT_UPDATE_PROGRESS, cleaned_progress, **kwargs) - - def trigger(self, event_name, *args, **kwargs): - """Execute all callbacks registered for the given event type. - - NOTE(harlowja): if a bound callback raises an exception it will be - logged (at a ``WARNING`` level) and the exception - will be dropped. - - :param event_name: event name to trigger - :param args: arbitrary positional arguments passed to the triggered - callbacks (if any are matched), these will be in addition - to any ``kwargs`` provided on binding (these are passed - as positional arguments to the callback). - :param kwargs: arbitrary keyword arguments passed to the triggered - callbacks (if any are matched), these will be in addition - to any ``kwargs`` provided on binding (these are passed - as keyword arguments to the callback). - """ - for (cb, event_data) in self._events_listeners.get(event_name, []): - try: - cb(self, event_data, *args, **kwargs) - except Exception: - LOG.warn("Failed calling callback `%s` on event '%s'", - reflection.get_callable_name(cb), event_name, - exc_info=True) - - @contextlib.contextmanager - def autobind(self, event_name, callback, **kwargs): - """Binds & unbinds a given callback to the task. - - This function binds and unbinds using the context manager protocol. - When events are triggered on the task of the given event name this - callback will automatically be called with the provided - keyword arguments as the first argument (further arguments may be - provided by the entity triggering the event). - - The arguments are interpreted as for :func:`bind() `. - """ - bound = False - if callback is not None: - try: - self.bind(event_name, callback, **kwargs) - bound = True - except ValueError: - LOG.warn("Failed binding callback `%s` as a receiver of" - " event '%s' notifications emitted from task '%s'", - reflection.get_callable_name(callback), event_name, - self, exc_info=True) - try: - yield self - finally: - if bound: - self.unbind(event_name, callback) - - def bind(self, event_name, callback, **kwargs): - """Attach a callback to be triggered on a task event. - - Callbacks should *not* be bound, modified, or removed after execution - has commenced (they may be adjusted after execution has finished). This - is primarily due to the need to preserve the callbacks that exist at - execution time for engines which run tasks remotely or out of - process (so that those engines can correctly proxy back transmitted - events). - - Callbacks should also be *quick* to execute so that the engine calling - them can continue execution in a timely manner (if long running - callbacks need to exist, consider creating a separate pool + queue - for those that the attached callbacks put long running operations into - for execution by other entities). - - :param event_name: event type name - :param callback: callable to execute each time event is triggered - :param kwargs: optional named parameters that will be passed to the - callable object as a dictionary to the callbacks - *second* positional parameter. - :raises ValueError: if invalid event type, or callback is passed - """ - if event_name not in self.TASK_EVENTS: - raise ValueError("Unknown task event '%s', can only bind" - " to events %s" % (event_name, self.TASK_EVENTS)) - if callback is not None: - if not six.callable(callback): - raise ValueError("Event handler callback must be callable") - self._events_listeners[event_name].append((callback, kwargs)) - - def unbind(self, event_name, callback=None): - """Remove a previously-attached event callback from the task. - - If a callback is not passed, then this will unbind *all* event - callbacks for the provided event. If multiple of the same callbacks - are bound, then the first match is removed (and only the first match). - - :param event_name: event type - :param callback: callback previously bound - - :rtype: boolean - :return: whether anything was removed - """ - removed_any = False - if not callback: - removed_any = self._events_listeners.pop(event_name, removed_any) - else: - event_listeners = self._events_listeners.get(event_name, []) - for i, (cb, _event_data) in enumerate(event_listeners): - if reflection.is_same_callback(cb, callback): - # NOTE(harlowja): its safe to do this as long as we stop - # iterating after we do the removal, otherwise its not - # safe (since this could have resized the list). - event_listeners.pop(i) - removed_any = True - break - return bool(removed_any) - - def listeners_iter(self): - """Return an iterator over the mapping of event => callbacks bound.""" - for event_name in list(six.iterkeys(self._events_listeners)): - # Use get() just incase it was removed while iterating... - event_listeners = self._events_listeners.get(event_name, []) - if event_listeners: - yield (event_name, event_listeners[:]) + self._notifier.notify(EVENT_UPDATE_PROGRESS, + {'progress': cleaned_progress}) class Task(BaseTask): diff --git a/taskflow/tests/unit/test_notifier.py b/taskflow/tests/unit/test_notifier.py index d9d40001..3e2a0633 100644 --- a/taskflow/tests/unit/test_notifier.py +++ b/taskflow/tests/unit/test_notifier.py @@ -84,6 +84,31 @@ class NotifierTest(test.TestCase): self.assertRaises(ValueError, notifier.register, nt.Notifier.ANY, 2) + def test_restricted_notifier(self): + notifier = nt.RestrictedNotifier(['a', 'b']) + self.assertRaises(ValueError, notifier.register, + 'c', lambda *args, **kargs: None) + notifier.register('b', lambda *args, **kargs: None) + self.assertEqual(1, len(notifier)) + + def test_restricted_notifier_any(self): + notifier = nt.RestrictedNotifier(['a', 'b']) + self.assertRaises(ValueError, notifier.register, + 'c', lambda *args, **kargs: None) + notifier.register('b', lambda *args, **kargs: None) + self.assertEqual(1, len(notifier)) + notifier.register(nt.RestrictedNotifier.ANY, + lambda *args, **kargs: None) + self.assertEqual(2, len(notifier)) + + def test_restricted_notifier_no_any(self): + notifier = nt.RestrictedNotifier(['a', 'b'], allow_any=False) + self.assertRaises(ValueError, notifier.register, + nt.RestrictedNotifier.ANY, + lambda *args, **kargs: None) + notifier.register('b', lambda *args, **kargs: None) + self.assertEqual(1, len(notifier)) + def test_selective_notify(self): call_counts = collections.defaultdict(list) diff --git a/taskflow/tests/unit/test_progress.py b/taskflow/tests/unit/test_progress.py index f37d1132..943f93c0 100644 --- a/taskflow/tests/unit/test_progress.py +++ b/taskflow/tests/unit/test_progress.py @@ -39,7 +39,12 @@ class ProgressTask(task.Task): class ProgressTaskWithDetails(task.Task): def execute(self): - self.update_progress(0.5, test='test data', foo='bar') + details = { + 'progress': 0.5, + 'test': 'test data', + 'foo': 'bar', + } + self.notifier.notify(task.EVENT_UPDATE_PROGRESS, details) class TestProgress(test.TestCase): @@ -60,12 +65,12 @@ class TestProgress(test.TestCase): def test_sanity_progress(self): fired_events = [] - def notify_me(task, event_data, progress): - fired_events.append(progress) + def notify_me(event_type, details): + fired_events.append(details.pop('progress')) ev_count = 5 t = ProgressTask("test", ev_count) - t.bind('update_progress', notify_me) + t.notifier.register(task.EVENT_UPDATE_PROGRESS, notify_me) flo = lf.Flow("test") flo.add(t) e = self._make_engine(flo) @@ -77,11 +82,11 @@ class TestProgress(test.TestCase): def test_no_segments_progress(self): fired_events = [] - def notify_me(task, event_data, progress): - fired_events.append(progress) + def notify_me(event_type, details): + fired_events.append(details.pop('progress')) t = ProgressTask("test", 0) - t.bind('update_progress', notify_me) + t.notifier.register(task.EVENT_UPDATE_PROGRESS, notify_me) flo = lf.Flow("test") flo.add(t) e = self._make_engine(flo) @@ -121,12 +126,12 @@ class TestProgress(test.TestCase): def test_dual_storage_progress(self): fired_events = [] - def notify_me(task, event_data, progress): - fired_events.append(progress) + def notify_me(event_type, details): + fired_events.append(details.pop('progress')) with contextlib.closing(impl_memory.MemoryBackend({})) as be: t = ProgressTask("test", 5) - t.bind('update_progress', notify_me) + t.notifier.register(task.EVENT_UPDATE_PROGRESS, notify_me) flo = lf.Flow("test") flo.add(t) b, fd = p_utils.temporary_flow_detail(be) diff --git a/taskflow/tests/unit/test_task.py b/taskflow/tests/unit/test_task.py index b80c0c6a..2f415840 100644 --- a/taskflow/tests/unit/test_task.py +++ b/taskflow/tests/unit/test_task.py @@ -17,7 +17,7 @@ from taskflow import task from taskflow import test from taskflow.test import mock -from taskflow.utils import reflection +from taskflow.types import notifier class MyTask(task.Task): @@ -198,24 +198,24 @@ class TaskTest(test.TestCase): values = [0.0, 0.5, 1.0] result = [] - def progress_callback(task, event_data, progress): - result.append(progress) + def progress_callback(event_type, details): + result.append(details.pop('progress')) a_task = ProgressTask() - with a_task.autobind(task.EVENT_UPDATE_PROGRESS, progress_callback): - a_task.execute(values) + a_task.notifier.register(task.EVENT_UPDATE_PROGRESS, progress_callback) + a_task.execute(values) self.assertEqual(result, values) @mock.patch.object(task.LOG, 'warn') def test_update_progress_lower_bound(self, mocked_warn): result = [] - def progress_callback(task, event_data, progress): - result.append(progress) + def progress_callback(event_type, details): + result.append(details.pop('progress')) a_task = ProgressTask() - with a_task.autobind(task.EVENT_UPDATE_PROGRESS, progress_callback): - a_task.execute([-1.0, -0.5, 0.0]) + a_task.notifier.register(task.EVENT_UPDATE_PROGRESS, progress_callback) + a_task.execute([-1.0, -0.5, 0.0]) self.assertEqual(result, [0.0, 0.0, 0.0]) self.assertEqual(mocked_warn.call_count, 2) @@ -223,81 +223,87 @@ class TaskTest(test.TestCase): def test_update_progress_upper_bound(self, mocked_warn): result = [] - def progress_callback(task, event_data, progress): - result.append(progress) + def progress_callback(event_type, details): + result.append(details.pop('progress')) a_task = ProgressTask() - with a_task.autobind(task.EVENT_UPDATE_PROGRESS, progress_callback): - a_task.execute([1.0, 1.5, 2.0]) + a_task.notifier.register(task.EVENT_UPDATE_PROGRESS, progress_callback) + a_task.execute([1.0, 1.5, 2.0]) self.assertEqual(result, [1.0, 1.0, 1.0]) self.assertEqual(mocked_warn.call_count, 2) - @mock.patch.object(task.LOG, 'warn') + @mock.patch.object(notifier.LOG, 'warn') def test_update_progress_handler_failure(self, mocked_warn): + def progress_callback(*args, **kwargs): raise Exception('Woot!') a_task = ProgressTask() - with a_task.autobind(task.EVENT_UPDATE_PROGRESS, progress_callback): - a_task.execute([0.5]) - mocked_warn.assert_called_once_with( - mock.ANY, reflection.get_callable_name(progress_callback), - task.EVENT_UPDATE_PROGRESS, exc_info=mock.ANY) + a_task.notifier.register(task.EVENT_UPDATE_PROGRESS, progress_callback) + a_task.execute([0.5]) + mocked_warn.assert_called_once() - def test_autobind_handler_is_none(self): + def test_register_handler_is_none(self): a_task = MyTask() - with a_task.autobind(task.EVENT_UPDATE_PROGRESS, None): - self.assertEqual(len(list(a_task.listeners_iter())), 0) + self.assertRaises(ValueError, a_task.notifier.register, + task.EVENT_UPDATE_PROGRESS, None) + self.assertEqual(len(a_task.notifier), 0) - def test_unbind_any_handler(self): + def test_deregister_any_handler(self): a_task = MyTask() - self.assertEqual(len(list(a_task.listeners_iter())), 0) - a_task.bind(task.EVENT_UPDATE_PROGRESS, lambda: None) - self.assertEqual(len(list(a_task.listeners_iter())), 1) - self.assertTrue(a_task.unbind(task.EVENT_UPDATE_PROGRESS)) - self.assertEqual(len(list(a_task.listeners_iter())), 0) + self.assertEqual(len(a_task.notifier), 0) + a_task.notifier.register(task.EVENT_UPDATE_PROGRESS, + lambda event_type, details: None) + self.assertEqual(len(a_task.notifier), 1) + a_task.notifier.deregister_event(task.EVENT_UPDATE_PROGRESS) + self.assertEqual(len(a_task.notifier), 0) - def test_unbind_any_handler_empty_listeners(self): + def test_deregister_any_handler_empty_listeners(self): a_task = MyTask() - self.assertEqual(len(list(a_task.listeners_iter())), 0) - self.assertFalse(a_task.unbind(task.EVENT_UPDATE_PROGRESS)) - self.assertEqual(len(list(a_task.listeners_iter())), 0) + self.assertEqual(len(a_task.notifier), 0) + self.assertFalse(a_task.notifier.deregister_event( + task.EVENT_UPDATE_PROGRESS)) + self.assertEqual(len(a_task.notifier), 0) - def test_unbind_non_existent_listener(self): - handler1 = lambda: None - handler2 = lambda: None + def test_deregister_non_existent_listener(self): + handler1 = lambda event_type, details: None + handler2 = lambda event_type, details: None a_task = MyTask() - a_task.bind(task.EVENT_UPDATE_PROGRESS, handler1) - self.assertEqual(len(list(a_task.listeners_iter())), 1) - self.assertFalse(a_task.unbind(task.EVENT_UPDATE_PROGRESS, handler2)) - self.assertEqual(len(list(a_task.listeners_iter())), 1) + a_task.notifier.register(task.EVENT_UPDATE_PROGRESS, handler1) + self.assertEqual(len(list(a_task.notifier.listeners_iter())), 1) + a_task.notifier.deregister(task.EVENT_UPDATE_PROGRESS, handler2) + self.assertEqual(len(list(a_task.notifier.listeners_iter())), 1) + a_task.notifier.deregister(task.EVENT_UPDATE_PROGRESS, handler1) + self.assertEqual(len(list(a_task.notifier.listeners_iter())), 0) def test_bind_not_callable(self): - task = MyTask() - self.assertRaises(ValueError, task.bind, 'update_progress', 2) + a_task = MyTask() + self.assertRaises(ValueError, a_task.notifier.register, + task.EVENT_UPDATE_PROGRESS, 2) def test_copy_no_listeners(self): - handler1 = lambda: None + handler1 = lambda event_type, details: None a_task = MyTask() - a_task.bind(task.EVENT_UPDATE_PROGRESS, handler1) + a_task.notifier.register(task.EVENT_UPDATE_PROGRESS, handler1) b_task = a_task.copy(retain_listeners=False) - self.assertEqual(len(list(a_task.listeners_iter())), 1) - self.assertEqual(len(list(b_task.listeners_iter())), 0) + self.assertEqual(len(a_task.notifier), 1) + self.assertEqual(len(b_task.notifier), 0) def test_copy_listeners(self): - handler1 = lambda: None - handler2 = lambda: None + handler1 = lambda event_type, details: None + handler2 = lambda event_type, details: None a_task = MyTask() - a_task.bind(task.EVENT_UPDATE_PROGRESS, handler1) + a_task.notifier.register(task.EVENT_UPDATE_PROGRESS, handler1) b_task = a_task.copy() - self.assertEqual(len(list(b_task.listeners_iter())), 1) - self.assertTrue(a_task.unbind(task.EVENT_UPDATE_PROGRESS)) - self.assertEqual(len(list(a_task.listeners_iter())), 0) - self.assertEqual(len(list(b_task.listeners_iter())), 1) - b_task.bind(task.EVENT_UPDATE_PROGRESS, handler2) - listeners = dict(list(b_task.listeners_iter())) + self.assertEqual(len(b_task.notifier), 1) + self.assertTrue(a_task.notifier.deregister_event( + task.EVENT_UPDATE_PROGRESS)) + self.assertEqual(len(a_task.notifier), 0) + self.assertEqual(len(b_task.notifier), 1) + b_task.notifier.register(task.EVENT_UPDATE_PROGRESS, handler2) + listeners = dict(list(b_task.notifier.listeners_iter())) self.assertEqual(len(listeners[task.EVENT_UPDATE_PROGRESS]), 2) - self.assertEqual(len(list(a_task.listeners_iter())), 0) + self.assertEqual(len(a_task.notifier), 0) class FunctorTaskTest(test.TestCase): diff --git a/taskflow/tests/unit/worker_based/test_endpoint.py b/taskflow/tests/unit/worker_based/test_endpoint.py index 2a52f5ab..36abb980 100644 --- a/taskflow/tests/unit/worker_based/test_endpoint.py +++ b/taskflow/tests/unit/worker_based/test_endpoint.py @@ -42,14 +42,14 @@ class TestEndpoint(test.TestCase): self.task_result = 1 def test_creation(self): - task = self.task_ep._get_task() + task = self.task_ep.generate() self.assertEqual(self.task_ep.name, self.task_cls_name) self.assertIsInstance(task, self.task_cls) self.assertEqual(task.name, self.task_cls_name) def test_creation_with_task_name(self): task_name = 'test' - task = self.task_ep._get_task(name=task_name) + task = self.task_ep.generate(name=task_name) self.assertEqual(self.task_ep.name, self.task_cls_name) self.assertIsInstance(task, self.task_cls) self.assertEqual(task.name, task_name) @@ -58,20 +58,22 @@ class TestEndpoint(test.TestCase): # NOTE(skudriashev): Exception is expected here since task # is created without any arguments passing to its constructor. endpoint = ep.Endpoint(Task) - self.assertRaises(TypeError, endpoint._get_task) + self.assertRaises(TypeError, endpoint.generate) def test_to_str(self): self.assertEqual(str(self.task_ep), self.task_cls_name) def test_execute(self): - result = self.task_ep.execute(task_name=self.task_cls_name, + task = self.task_ep.generate(self.task_cls_name) + result = self.task_ep.execute(task, task_uuid=self.task_uuid, arguments=self.task_args, progress_callback=None) self.assertEqual(result, self.task_result) def test_revert(self): - result = self.task_ep.revert(task_name=self.task_cls_name, + task = self.task_ep.generate(self.task_cls_name) + result = self.task_ep.revert(task, task_uuid=self.task_uuid, arguments=self.task_args, progress_callback=None, diff --git a/taskflow/tests/unit/worker_based/test_executor.py b/taskflow/tests/unit/worker_based/test_executor.py index ba73ad7e..55834214 100644 --- a/taskflow/tests/unit/worker_based/test_executor.py +++ b/taskflow/tests/unit/worker_based/test_executor.py @@ -21,6 +21,7 @@ from oslo.utils import timeutils from taskflow.engines.worker_based import executor from taskflow.engines.worker_based import protocol as pr +from taskflow import task as task_atom from taskflow import test from taskflow.test import mock from taskflow.tests import utils as test_utils @@ -102,13 +103,18 @@ class TestWorkerTaskExecutor(test.MockTestCase): self.assertEqual(expected_calls, self.request_inst_mock.mock_calls) def test_on_message_response_state_progress(self): - response = pr.Response(pr.PROGRESS, progress=1.0) + response = pr.Response(pr.EVENT, + event_type=task_atom.EVENT_UPDATE_PROGRESS, + details={'progress': 1.0}) ex = self.executor() ex._requests_cache[self.task_uuid] = self.request_inst_mock ex._process_response(response.to_dict(), self.message_mock) - self.assertEqual(self.request_inst_mock.mock_calls, - [mock.call.on_progress(progress=1.0)]) + expected_calls = [ + mock.call.notifier.notify(task_atom.EVENT_UPDATE_PROGRESS, + {'progress': 1.0}), + ] + self.assertEqual(expected_calls, self.request_inst_mock.mock_calls) def test_on_message_response_state_failure(self): a_failure = failure.Failure.from_exception(Exception('test')) @@ -211,7 +217,7 @@ class TestWorkerTaskExecutor(test.MockTestCase): expected_calls = [ mock.call.Request(self.task, self.task_uuid, 'execute', - self.task_args, None, self.timeout), + self.task_args, self.timeout), mock.call.request.transition_and_log_error(pr.PENDING, logger=mock.ANY), mock.call.proxy.publish(self.request_inst_mock, @@ -231,7 +237,7 @@ class TestWorkerTaskExecutor(test.MockTestCase): expected_calls = [ mock.call.Request(self.task, self.task_uuid, 'revert', - self.task_args, None, self.timeout, + self.task_args, self.timeout, failures=self.task_failures, result=self.task_result), mock.call.request.transition_and_log_error(pr.PENDING, @@ -250,7 +256,7 @@ class TestWorkerTaskExecutor(test.MockTestCase): expected_calls = [ mock.call.Request(self.task, self.task_uuid, 'execute', - self.task_args, None, self.timeout) + self.task_args, self.timeout), ] self.assertEqual(self.master_mock.mock_calls, expected_calls) @@ -264,7 +270,7 @@ class TestWorkerTaskExecutor(test.MockTestCase): expected_calls = [ mock.call.Request(self.task, self.task_uuid, 'execute', - self.task_args, None, self.timeout), + self.task_args, self.timeout), mock.call.request.transition_and_log_error(pr.PENDING, logger=mock.ANY), mock.call.proxy.publish(self.request_inst_mock, diff --git a/taskflow/tests/unit/worker_based/test_message_pump.py b/taskflow/tests/unit/worker_based/test_message_pump.py index 8b48b435..cae4fa5c 100644 --- a/taskflow/tests/unit/worker_based/test_message_pump.py +++ b/taskflow/tests/unit/worker_based/test_message_pump.py @@ -118,7 +118,7 @@ class TestMessagePump(test.TestCase): else: p.publish(pr.Request(test_utils.DummyTask("dummy_%s" % i), uuidutils.generate_uuid(), - pr.EXECUTE, [], None, None), TEST_TOPIC) + pr.EXECUTE, [], None), TEST_TOPIC) self.assertTrue(barrier.wait(test_utils.WAIT_TIMEOUT)) self.assertEqual(0, barrier.needed) diff --git a/taskflow/tests/unit/worker_based/test_protocol.py b/taskflow/tests/unit/worker_based/test_protocol.py index 2f5fd927..d2f2cc02 100644 --- a/taskflow/tests/unit/worker_based/test_protocol.py +++ b/taskflow/tests/unit/worker_based/test_protocol.py @@ -22,7 +22,6 @@ from taskflow.engines.worker_based import protocol as pr from taskflow import exceptions as excp from taskflow.openstack.common import uuidutils from taskflow import test -from taskflow.test import mock from taskflow.tests import utils from taskflow.types import failure @@ -53,7 +52,7 @@ class TestProtocolValidation(test.TestCase): def test_request(self): msg = pr.Request(utils.DummyTask("hi"), uuidutils.generate_uuid(), - pr.EXECUTE, {}, None, 1.0) + pr.EXECUTE, {}, 1.0) pr.Request.validate(msg.to_dict()) def test_request_invalid(self): @@ -66,13 +65,14 @@ class TestProtocolValidation(test.TestCase): def test_request_invalid_action(self): msg = pr.Request(utils.DummyTask("hi"), uuidutils.generate_uuid(), - pr.EXECUTE, {}, None, 1.0) + pr.EXECUTE, {}, 1.0) msg = msg.to_dict() msg['action'] = 'NOTHING' self.assertRaises(excp.InvalidFormat, pr.Request.validate, msg) def test_response_progress(self): - msg = pr.Response(pr.PROGRESS, progress=0.5, event_data={}) + msg = pr.Response(pr.EVENT, details={'progress': 0.5}, + event_type='blah') pr.Response.validate(msg.to_dict()) def test_response_completion(self): @@ -80,7 +80,9 @@ class TestProtocolValidation(test.TestCase): pr.Response.validate(msg.to_dict()) def test_response_mixed_invalid(self): - msg = pr.Response(pr.PROGRESS, progress=0.5, event_data={}, result=1) + msg = pr.Response(pr.EVENT, + details={'progress': 0.5}, + event_type='blah', result=1) self.assertRaises(excp.InvalidFormat, pr.Response.validate, msg) def test_response_bad_state(self): @@ -184,16 +186,3 @@ class TestProtocol(test.TestCase): request.set_result(111) result = request.result.result() self.assertEqual(result, (executor.EXECUTED, 111)) - - def test_on_progress(self): - progress_callback = mock.MagicMock(name='progress_callback') - request = self.request(task=self.task, - progress_callback=progress_callback) - request.on_progress('event_data', 0.0) - request.on_progress('event_data', 1.0) - - expected_calls = [ - mock.call(self.task, 'event_data', 0.0), - mock.call(self.task, 'event_data', 1.0) - ] - self.assertEqual(progress_callback.mock_calls, expected_calls) diff --git a/taskflow/tests/unit/worker_based/test_server.py b/taskflow/tests/unit/worker_based/test_server.py index 5e9129ae..7da5b432 100644 --- a/taskflow/tests/unit/worker_based/test_server.py +++ b/taskflow/tests/unit/worker_based/test_server.py @@ -19,6 +19,7 @@ import six from taskflow.engines.worker_based import endpoint as ep from taskflow.engines.worker_based import protocol as pr from taskflow.engines.worker_based import server +from taskflow import task as task_atom from taskflow import test from taskflow.test import mock from taskflow.tests import utils @@ -103,45 +104,41 @@ class TestServer(test.MockTestCase): def test_parse_request(self): request = self.make_request() - task_cls, action, task_args = server.Server._parse_request(**request) - - self.assertEqual((task_cls, action, task_args), - (self.task.name, self.task_action, - dict(task_name=self.task.name, - arguments=self.task_args))) + bundle = server.Server._parse_request(**request) + task_cls, task_name, action, task_args = bundle + self.assertEqual((task_cls, task_name, action, task_args), + (self.task.name, self.task.name, self.task_action, + dict(arguments=self.task_args))) def test_parse_request_with_success_result(self): request = self.make_request(action='revert', result=1) - task_cls, action, task_args = server.Server._parse_request(**request) - - self.assertEqual((task_cls, action, task_args), - (self.task.name, 'revert', - dict(task_name=self.task.name, - arguments=self.task_args, + bundle = server.Server._parse_request(**request) + task_cls, task_name, action, task_args = bundle + self.assertEqual((task_cls, task_name, action, task_args), + (self.task.name, self.task.name, 'revert', + dict(arguments=self.task_args, result=1))) def test_parse_request_with_failure_result(self): a_failure = failure.Failure.from_exception(Exception('test')) request = self.make_request(action='revert', result=a_failure) - task_cls, action, task_args = server.Server._parse_request(**request) - - self.assertEqual((task_cls, action, task_args), - (self.task.name, 'revert', - dict(task_name=self.task.name, - arguments=self.task_args, + bundle = server.Server._parse_request(**request) + task_cls, task_name, action, task_args = bundle + self.assertEqual((task_cls, task_name, action, task_args), + (self.task.name, self.task.name, 'revert', + dict(arguments=self.task_args, result=utils.FailureMatcher(a_failure)))) def test_parse_request_with_failures(self): failures = {'0': failure.Failure.from_exception(Exception('test1')), '1': failure.Failure.from_exception(Exception('test2'))} request = self.make_request(action='revert', failures=failures) - task_cls, action, task_args = server.Server._parse_request(**request) - + bundle = server.Server._parse_request(**request) + task_cls, task_name, action, task_args = bundle self.assertEqual( - (task_cls, action, task_args), - (self.task.name, 'revert', - dict(task_name=self.task.name, - arguments=self.task_args, + (task_cls, task_name, action, task_args), + (self.task.name, self.task.name, 'revert', + dict(arguments=self.task_args, failures=dict((i, utils.FailureMatcher(f)) for i, f in six.iteritems(failures))))) @@ -182,17 +179,19 @@ class TestServer(test.MockTestCase): mock.call.Response(pr.RUNNING), mock.call.proxy.publish(self.response_inst_mock, self.reply_to, correlation_id=self.task_uuid), - mock.call.Response(pr.PROGRESS, progress=0.0, event_data={}), + mock.call.Response(pr.EVENT, details={'progress': 0.0}, + event_type=task_atom.EVENT_UPDATE_PROGRESS), mock.call.proxy.publish(self.response_inst_mock, self.reply_to, correlation_id=self.task_uuid), - mock.call.Response(pr.PROGRESS, progress=1.0, event_data={}), + mock.call.Response(pr.EVENT, details={'progress': 1.0}, + event_type=task_atom.EVENT_UPDATE_PROGRESS), mock.call.proxy.publish(self.response_inst_mock, self.reply_to, correlation_id=self.task_uuid), mock.call.Response(pr.SUCCESS, result=5), mock.call.proxy.publish(self.response_inst_mock, self.reply_to, correlation_id=self.task_uuid) ] - self.assertEqual(self.master_mock.mock_calls, master_mock_calls) + self.assertEqual(master_mock_calls, self.master_mock.mock_calls) def test_process_request(self): # create server and process request diff --git a/taskflow/types/notifier.py b/taskflow/types/notifier.py index 8e58302f..ff6a8ae3 100644 --- a/taskflow/types/notifier.py +++ b/taskflow/types/notifier.py @@ -15,6 +15,7 @@ # under the License. import collections +import copy import logging import six @@ -39,6 +40,14 @@ class _Listener(object): else: self._kwargs = kwargs.copy() + @property + def kwargs(self): + return self._kwargs + + @property + def args(self): + return self._args + def __call__(self, event_type, details): if self._details_filter is not None: if not self._details_filter(details): @@ -117,17 +126,18 @@ class Notifier(object): event type will be called. :param event_type: event type that occurred - :param details: addition event details + :param details: additional event details *dictionary* passed to + callback keyword argument with the same name. """ listeners = list(self._listeners.get(self.ANY, [])) - for listener in self._listeners[event_type]: - if listener not in listeners: - listeners.append(listener) + listeners.extend(self._listeners.get(event_type, [])) if not listeners: return + if not details: + details = {} for listener in listeners: try: - listener(event_type, details) + listener(event_type, details.copy()) except Exception: LOG.warn("Failure calling listener %s to notify about event" " %s, details: %s", listener, event_type, @@ -150,6 +160,9 @@ class Notifier(object): if details_filter is not None: if not six.callable(details_filter): raise ValueError("Details filter must be callable") + if not self.can_be_registered(event_type): + raise ValueError("Disallowed event type '%s' can not have a" + " callback registered" % event_type) if self.is_registered(event_type, callback, details_filter=details_filter): raise ValueError("Event callback already registered with" @@ -165,10 +178,61 @@ class Notifier(object): details_filter=details_filter)) def deregister(self, event_type, callback, details_filter=None): - """Remove a single callback from listening to event ``event_type``.""" + """Remove a single listener bound to event ``event_type``.""" if event_type not in self._listeners: - return - for i, listener in enumerate(self._listeners[event_type]): + return False + for i, listener in enumerate(self._listeners.get(event_type, [])): if listener.is_equivalent(callback, details_filter=details_filter): self._listeners[event_type].pop(i) - break + return True + return False + + def deregister_event(self, event_type): + """Remove a group of listeners bound to event ``event_type``.""" + return len(self._listeners.pop(event_type, [])) + + def copy(self): + c = copy.copy(self) + c._listeners = collections.defaultdict(list) + for event_type, listeners in six.iteritems(self._listeners): + c._listeners[event_type] = listeners[:] + return c + + def listeners_iter(self): + """Return an iterator over the mapping of event => listeners bound.""" + for event_type, listeners in six.iteritems(self._listeners): + if listeners: + yield (event_type, listeners) + + def can_be_registered(self, event_type): + """Checks if the event can be registered/subscribed to.""" + return True + + +class RestrictedNotifier(Notifier): + """A notification class that restricts events registered/triggered. + + NOTE(harlowja): This class unlike :class:`.Notifier` restricts and + disallows registering callbacks for event types that are not declared + when constructing the notifier. + """ + + def __init__(self, watchable_events, allow_any=True): + super(RestrictedNotifier, self).__init__() + self._watchable_events = frozenset(watchable_events) + self._allow_any = allow_any + + def events_iter(self): + """Returns iterator of events that can be registered/subscribed to. + + NOTE(harlowja): does not include back the ``ANY`` event type as that + meta-type is not a specific event but is a capture-all that does not + imply the same meaning as specific event types. + """ + for event_type in self._watchable_events: + yield event_type + + def can_be_registered(self, event_type): + """Checks if the event can be registered/subscribed to.""" + return (event_type in self._watchable_events or + (event_type == self.ANY and self._allow_any)) From aa8d55d948d3e0c47898446fef88654299ad014e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 13 Dec 2014 23:09:19 -0800 Subject: [PATCH 155/240] Cleanup some doc warnings/bad/broken links This fixes some of the old links to classes that have been moved, or split, fixes some of the sphinx warnings that were being output and cleans up the reference to deprecated properties. Change-Id: Ib930c54bcdf15876093cbe5b6527a195b9594f40 --- doc/source/engines.rst | 17 ++++++++++------- doc/source/notifications.rst | 31 ++++++++++++++++++++++--------- taskflow/utils/reflection.py | 2 +- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/doc/source/engines.rst b/doc/source/engines.rst index e3680672..0c4b822f 100644 --- a/doc/source/engines.rst +++ b/doc/source/engines.rst @@ -167,7 +167,7 @@ Additional supported keyword arguments: * ``executor``: a object that implements a :pep:`3148` compatible `executor`_ interface; it will be used for scheduling tasks. You can use instances of a `thread pool executor`_ or a :py:class:`green executor - ` (which internally uses + ` (which internally uses `eventlet `_ and greenthread pools). .. tip:: @@ -271,13 +271,13 @@ Scheduling ^^^^^^^^^^ This stage selects which atoms are eligible to run by using a -:py:class:`~taskflow.engines.action_engine.runtime.Scheduler` implementation +:py:class:`~taskflow.engines.action_engine.scheduler.Scheduler` implementation (the default implementation looks at there intention, checking if predecessor atoms have ran and so-on, using a :py:class:`~taskflow.engines.action_engine.analyzer.Analyzer` helper object as needed) and submits those atoms to a previously provided compatible `executor`_ for asynchronous execution. This -:py:class:`~taskflow.engines.action_engine.runtime.Scheduler` will return a +:py:class:`~taskflow.engines.action_engine.scheduler.Scheduler` will return a `future`_ object for each atom scheduled; all of which are collected into a list of not done futures. This will end the initial round of scheduling and at this point the engine enters the :ref:`waiting ` stage. @@ -290,7 +290,7 @@ Waiting In this stage the engine waits for any of the future objects previously submitted to complete. Once one of the future objects completes (or fails) that atoms result will be examined and finalized using a -:py:class:`~taskflow.engines.action_engine.runtime.Completer` implementation. +:py:class:`~taskflow.engines.action_engine.completer.Completer` implementation. It typically will persist results to a provided persistence backend (saved into the corresponding :py:class:`~taskflow.persistence.logbook.AtomDetail` and :py:class:`~taskflow.persistence.logbook.FlowDetail` objects) and reflect @@ -330,18 +330,21 @@ Interfaces .. automodule:: taskflow.engines.action_engine.analyzer .. automodule:: taskflow.engines.action_engine.compiler +.. automodule:: taskflow.engines.action_engine.completer .. automodule:: taskflow.engines.action_engine.engine .. automodule:: taskflow.engines.action_engine.runner .. automodule:: taskflow.engines.action_engine.runtime +.. automodule:: taskflow.engines.action_engine.scheduler +.. automodule:: taskflow.engines.action_engine.scopes .. automodule:: taskflow.engines.base Hierarchy ========= .. inheritance-diagram:: - taskflow.engines.base - taskflow.engines.action_engine.engine - taskflow.engines.worker_based.engine + taskflow.engines.action_engine.engine.ActionEngine + taskflow.engines.base.Engine + taskflow.engines.worker_based.engine.WorkerBasedActionEngine :parts: 1 .. _future: https://docs.python.org/dev/library/concurrent.futures.html#future-objects diff --git a/doc/source/notifications.rst b/doc/source/notifications.rst index 3b857506..118b5aab 100644 --- a/doc/source/notifications.rst +++ b/doc/source/notifications.rst @@ -17,10 +17,9 @@ transitions, which is useful for monitoring, logging, metrics, debugging and plenty of other tasks. To receive these notifications you should register a callback with -an instance of the :py:class:`~taskflow.utils.misc.Notifier` -class that is attached -to :py:class:`Engine ` -attributes ``task_notifier`` and ``notifier``. +an instance of the :py:class:`~taskflow.types.notifier.Notifier` +class that is attached to :py:class:`~taskflow.engines.base.Engine` +attributes ``atom_notifier`` and ``notifier``. TaskFlow also comes with a set of predefined :ref:`listeners `, and provides means to write your own listeners, which can be more convenient than @@ -34,7 +33,7 @@ Flow notifications ------------------ To receive notification on flow state changes use the -:py:class:`~taskflow.utils.misc.Notifier` instance available as the +:py:class:`~taskflow.types.notifier.Notifier` instance available as the ``notifier`` property of an engine. A basic example is: @@ -69,8 +68,8 @@ Task notifications ------------------ To receive notification on task state changes use the -:py:class:`~taskflow.utils.misc.Notifier` instance available as the -``task_notifier`` property of an engine. +:py:class:`~taskflow.types.notifier.Notifier` instance available as the +``atom_notifier`` property of an engine. A basic example is: @@ -149,12 +148,12 @@ For example, this is how you can use Basic listener -------------- -.. autoclass:: taskflow.listeners.base.ListenerBase +.. autoclass:: taskflow.listeners.base.Listener Printing and logging listeners ------------------------------ -.. autoclass:: taskflow.listeners.base.LoggingBase +.. autoclass:: taskflow.listeners.base.DumpingListener .. autoclass:: taskflow.listeners.logging.LoggingListener @@ -173,3 +172,17 @@ Claim listener -------------- .. autoclass:: taskflow.listeners.claims.CheckingClaimListener + +Hierarchy +--------- + +.. inheritance-diagram:: + taskflow.listeners.base.DumpingListener + taskflow.listeners.base.Listener + taskflow.listeners.claims.CheckingClaimListener + taskflow.listeners.logging.DynamicLoggingListener + taskflow.listeners.logging.LoggingListener + taskflow.listeners.printing.PrintingListener + taskflow.listeners.timing.PrintingTimingListener + taskflow.listeners.timing.TimingListener + :parts: 1 diff --git a/taskflow/utils/reflection.py b/taskflow/utils/reflection.py index 34793e00..08eaf6c9 100644 --- a/taskflow/utils/reflection.py +++ b/taskflow/utils/reflection.py @@ -230,7 +230,7 @@ def _get_arg_spec(function): def get_callable_args(function, required_only=False): """Get names of callable arguments. - Special arguments (like *args and **kwargs) are not included into + Special arguments (like ``*args`` and ``**kwargs``) are not included into output. If required_only is True, optional arguments (with default values) From e9ecdc745d22470039228645838a2e4c1f3e1ba9 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 14 Dec 2014 18:22:35 -0800 Subject: [PATCH 156/240] Replace autobind with a notifier module helper function Instead of having the executor method provide a bind and unbind function just have the notifier module provide this, which allows those who were using the task autobind to use this helper as well as the execution code itself. Change-Id: If61c8c1669d7c0d66e2daab5fd773b8c7756f202 --- taskflow/engines/action_engine/executor.py | 25 ++++++---------------- taskflow/types/notifier.py | 23 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index afe37d1d..b7ff2f70 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -15,13 +15,13 @@ # under the License. import abc -import contextlib import six from taskflow import task as task_atom from taskflow.types import failure from taskflow.types import futures +from taskflow.types import notifier from taskflow.utils import async_utils from taskflow.utils import threading_utils @@ -30,23 +30,10 @@ EXECUTED = 'executed' REVERTED = 'reverted' -@contextlib.contextmanager -def _autobind(task, progress_callback=None): - bound = False - if progress_callback is not None: - task.notifier.register(task_atom.EVENT_UPDATE_PROGRESS, - progress_callback) - bound = True - try: - yield - finally: - if bound: - task.notifier.deregister(task_atom.EVENT_UPDATE_PROGRESS, - progress_callback) - - def _execute_task(task, arguments, progress_callback=None): - with _autobind(task, progress_callback=progress_callback): + with notifier.register_deregister(task.notifier, + task_atom.EVENT_UPDATE_PROGRESS, + callback=progress_callback): try: task.pre_execute() result = task.execute(**arguments) @@ -63,7 +50,9 @@ def _revert_task(task, arguments, result, failures, progress_callback=None): arguments = arguments.copy() arguments[task_atom.REVERT_RESULT] = result arguments[task_atom.REVERT_FLOW_FAILURES] = failures - with _autobind(task, progress_callback=progress_callback): + with notifier.register_deregister(task.notifier, + task_atom.EVENT_UPDATE_PROGRESS, + callback=progress_callback): try: task.pre_revert() result = task.revert(**arguments) diff --git a/taskflow/types/notifier.py b/taskflow/types/notifier.py index ff6a8ae3..6d8c63e9 100644 --- a/taskflow/types/notifier.py +++ b/taskflow/types/notifier.py @@ -15,6 +15,7 @@ # under the License. import collections +import contextlib import copy import logging @@ -236,3 +237,25 @@ class RestrictedNotifier(Notifier): """Checks if the event can be registered/subscribed to.""" return (event_type in self._watchable_events or (event_type == self.ANY and self._allow_any)) + + +@contextlib.contextmanager +def register_deregister(notifier, event_type, callback=None, + args=None, kwargs=None, details_filter=None): + """Context manager that registers a callback, then deregisters on exit. + + NOTE(harlowja): if the callback is none, then this registers nothing, which + is different from the behavior of the ``register`` method + which will *not* accept none as it is not callable... + """ + if callback is None: + yield + else: + notifier.register(event_type, callback, + args=args, kwargs=kwargs, + details_filter=details_filter) + try: + yield + finally: + notifier.deregister(event_type, callback, + details_filter=details_filter) From 97b4e18cc2b4e6c0ae7228ff70ef75dc4a5a1df7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 15 Dec 2014 15:47:58 -0800 Subject: [PATCH 157/240] Base task executor should provide 'wait_for_any' Instead of having each task executor reproduce the same code for 'wait_for_any' we can just have the base task implementation provide the function that everyone is replicating instead; making common code common and save the headache caused by the same code being in multiple places (which is bad for multiple reasons). Change-Id: Icea4b7e3df605ab11b17c248d05acb3f9c02a1ca --- taskflow/engines/action_engine/executor.py | 8 +------- taskflow/engines/worker_based/executor.py | 5 ----- taskflow/tests/unit/worker_based/test_executor.py | 6 +++--- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index b7ff2f70..002068b3 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -83,9 +83,9 @@ class TaskExecutor(object): progress_callback=None): """Schedules task reversion.""" - @abc.abstractmethod def wait_for_any(self, fs, timeout=None): """Wait for futures returned by this executor to complete.""" + return async_utils.wait_for_any(fs, timeout=timeout) def start(self): """Prepare to execute tasks.""" @@ -117,9 +117,6 @@ class SerialTaskExecutor(TaskExecutor): fut.atom = task return fut - def wait_for_any(self, fs, timeout=None): - return async_utils.wait_for_any(fs, timeout) - class ParallelTaskExecutor(TaskExecutor): """Executes tasks in parallel. @@ -148,9 +145,6 @@ class ParallelTaskExecutor(TaskExecutor): fut.atom = task return fut - def wait_for_any(self, fs, timeout=None): - return async_utils.wait_for_any(fs, timeout) - def start(self): if self._create_executor: if self._max_workers is not None: diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index cd068e81..bdef7bff 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -27,7 +27,6 @@ from taskflow import exceptions as exc from taskflow import logging from taskflow import task as task_atom from taskflow.types import timing as tt -from taskflow.utils import async_utils from taskflow.utils import misc from taskflow.utils import reflection from taskflow.utils import threading_utils as tu @@ -248,10 +247,6 @@ class WorkerTaskExecutor(executor.TaskExecutor): progress_callback, result=result, failures=failures) - def wait_for_any(self, fs, timeout=None): - """Wait for futures returned by this executor to complete.""" - return async_utils.wait_for_any(fs, timeout) - def wait_for_workers(self, workers=1, timeout=None): """Waits for geq workers to notify they are ready to do work. diff --git a/taskflow/tests/unit/worker_based/test_executor.py b/taskflow/tests/unit/worker_based/test_executor.py index 55834214..f075e2f8 100644 --- a/taskflow/tests/unit/worker_based/test_executor.py +++ b/taskflow/tests/unit/worker_based/test_executor.py @@ -58,7 +58,7 @@ class TestWorkerTaskExecutor(test.MockTestCase): self.request_inst_mock.expired = False self.request_inst_mock.task_cls = self.task.name self.wait_for_any_mock = self.patch( - 'taskflow.engines.worker_based.executor.async_utils.wait_for_any') + 'taskflow.engines.action_engine.executor.async_utils.wait_for_any') self.message_mock = mock.MagicMock(name='message') self.message_mock.properties = {'correlation_id': self.task_uuid, 'type': pr.RESPONSE} @@ -289,7 +289,7 @@ class TestWorkerTaskExecutor(test.MockTestCase): ex.wait_for_any(fs) expected_calls = [ - mock.call(fs, None) + mock.call(fs, timeout=None) ] self.assertEqual(self.wait_for_any_mock.mock_calls, expected_calls) @@ -300,7 +300,7 @@ class TestWorkerTaskExecutor(test.MockTestCase): ex.wait_for_any(fs, timeout) master_mock_calls = [ - mock.call(fs, timeout) + mock.call(fs, timeout=timeout) ] self.assertEqual(self.wait_for_any_mock.mock_calls, master_mock_calls) From 1b0618338c5eb70116cc056b31d83c18d90eac59 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 15 Dec 2014 16:37:15 -0800 Subject: [PATCH 158/240] Add a 'can_be_registered' method that checks before notifying Add a new method that can be used to check if a event type is allowed to trigger a notification; and initially use it to disallow the 'ANY' meta event type from being used to trigger notifications. Change-Id: I842fcc5d3e06f69a9479b60b3b89a24233171cfb --- taskflow/tests/unit/test_notifier.py | 14 ++++++++++++++ taskflow/types/notifier.py | 20 +++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/taskflow/tests/unit/test_notifier.py b/taskflow/tests/unit/test_notifier.py index 3e2a0633..60e0e1e8 100644 --- a/taskflow/tests/unit/test_notifier.py +++ b/taskflow/tests/unit/test_notifier.py @@ -38,6 +38,20 @@ class NotifierTest(test.TestCase): self.assertEqual(2, len(call_collector)) self.assertEqual(1, len(notifier)) + def test_notify_not_called(self): + call_collector = [] + + def call_me(state, details): + call_collector.append((state, details)) + + notifier = nt.Notifier() + notifier.register(nt.Notifier.ANY, call_me) + notifier.notify(nt.Notifier.ANY, {}) + self.assertFalse(notifier.can_trigger_notification(nt.Notifier.ANY)) + + self.assertEqual(0, len(call_collector)) + self.assertEqual(1, len(notifier)) + def test_notify_register_deregister(self): def call_me(state, details): diff --git a/taskflow/types/notifier.py b/taskflow/types/notifier.py index 6d8c63e9..98511fba 100644 --- a/taskflow/types/notifier.py +++ b/taskflow/types/notifier.py @@ -99,6 +99,9 @@ class Notifier(object): #: Kleene star constant that is used to recieve all notifications ANY = '*' + #: Events which can *not* be used to trigger notifications + _DISALLOWED_NOTIFICATION_EVENTS = set([ANY]) + def __init__(self): self._listeners = collections.defaultdict(list) @@ -124,12 +127,20 @@ class Notifier(object): """Notify about event occurrence. All callbacks registered to receive notifications about given - event type will be called. + event type will be called. If the provided event type can not be + used to emit notifications (this is checked via + the :meth:`.can_be_registered` method) then it will silently be + dropped (notification failures are not allowed to cause or + raise exceptions). :param event_type: event type that occurred :param details: additional event details *dictionary* passed to callback keyword argument with the same name. """ + if not self.can_trigger_notification(event_type): + LOG.debug("Event type '%s' is not allowed to trigger" + " notifications", event_type) + return listeners = list(self._listeners.get(self.ANY, [])) listeners.extend(self._listeners.get(event_type, [])) if not listeners: @@ -209,6 +220,13 @@ class Notifier(object): """Checks if the event can be registered/subscribed to.""" return True + def can_trigger_notification(self, event_type): + """Checks if the event can trigger a notification.""" + if event_type in self._DISALLOWED_NOTIFICATION_EVENTS: + return False + else: + return True + class RestrictedNotifier(Notifier): """A notification class that restricts events registered/triggered. From cafa3b2e256275d24d1c2a298580316448119740 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 7 Nov 2014 06:24:20 -0800 Subject: [PATCH 159/240] Add a parallel table mutation example A new simple example that is pretty easy to follow that does a embarrassingly parallel computation on some input table to create a new output table (by performing a multiplication on each cell in that source table to create a new table). Part of blueprint more-examples Change-Id: I2684f39b3525ee2d43a03ab353d029fdc0e1b2a1 --- doc/source/examples.rst | 12 ++ taskflow/examples/parallel_table_multiply.py | 129 +++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 taskflow/examples/parallel_table_multiply.py diff --git a/doc/source/examples.rst b/doc/source/examples.rst index 24a43dd0..61a21c2d 100644 --- a/doc/source/examples.rst +++ b/doc/source/examples.rst @@ -58,6 +58,18 @@ Watching execution timing :linenos: :lines: 16- +Table multiplier (in parallel) +============================== + +.. note:: + + Full source located at :example:`parallel_table_multiply` + +.. literalinclude:: ../../taskflow/examples/parallel_table_multiply.py + :language: python + :linenos: + :lines: 16- + Linear equation solver (explicit dependencies) ============================================== diff --git a/taskflow/examples/parallel_table_multiply.py b/taskflow/examples/parallel_table_multiply.py new file mode 100644 index 00000000..88562a2e --- /dev/null +++ b/taskflow/examples/parallel_table_multiply.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import csv +import logging +import os +import random +import sys + +logging.basicConfig(level=logging.ERROR) + +top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, + os.pardir)) +sys.path.insert(0, top_dir) + +from six.moves import range as compat_range + +from taskflow import engines +from taskflow.patterns import unordered_flow as uf +from taskflow import task +from taskflow.types import futures +from taskflow.utils import async_utils + +# INTRO: This example walks through a miniature workflow which does a parallel +# table modification where each row in the table gets adjusted by a thread, or +# green thread (if eventlet is available) in parallel and then the result +# is reformed into a new table and some verifications are performed on it +# to ensure everything went as expected. + + +MULTIPLER = 10 + + +class RowMultiplier(task.Task): + """Performs a modification of an input row, creating a output row.""" + + def __init__(self, name, index, row, multiplier): + super(RowMultiplier, self).__init__(name=name) + self.index = index + self.multiplier = multiplier + self.row = row + + def execute(self): + return [r * self.multiplier for r in self.row] + + +def make_flow(table): + # This creation will allow for parallel computation (since the flow here + # is specifically unordered; and when things are unordered they have + # no dependencies and when things have no dependencies they can just be + # ran at the same time, limited in concurrency by the executor or max + # workers of that executor...) + f = uf.Flow("root") + for i, row in enumerate(table): + f.add(RowMultiplier("m-%s" % i, i, row, MULTIPLER)) + # NOTE(harlowja): at this point nothing has ran, the above is just + # defining what should be done (but not actually doing it) and associating + # an ordering dependencies that should be enforced (the flow pattern used + # forces this), the engine in the later main() function will actually + # perform this work... + return f + + +def main(): + if len(sys.argv) == 2: + tbl = [] + with open(sys.argv[1], 'rb') as fh: + reader = csv.reader(fh) + for row in reader: + tbl.append([float(r) if r else 0.0 for r in row]) + else: + # Make some random table out of thin air... + tbl = [] + cols = random.randint(1, 100) + rows = random.randint(1, 100) + for _i in compat_range(0, rows): + row = [] + for _j in compat_range(0, cols): + row.append(random.random()) + tbl.append(row) + + # Generate the work to be done. + f = make_flow(tbl) + + # Now run it (using the specified executor)... + if async_utils.EVENTLET_AVAILABLE: + executor = futures.GreenThreadPoolExecutor(max_workers=5) + else: + executor = futures.ThreadPoolExecutor(max_workers=5) + try: + e = engines.load(f, engine='parallel', executor=executor) + for st in e.run_iter(): + print(st) + finally: + executor.shutdown() + + # Find the old rows and put them into place... + # + # TODO(harlowja): probably easier just to sort instead of search... + computed_tbl = [] + for i in compat_range(0, len(tbl)): + for t in f: + if t.index == i: + computed_tbl.append(e.storage.get(t.name)) + + # Do some basic validation (which causes the return code of this process + # to be different if things were not as expected...) + if len(computed_tbl) != len(tbl): + return 1 + else: + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 6520b9c35e1fe2f5a851b0beb9e337595fb58874 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 21 Oct 2014 17:21:52 -0700 Subject: [PATCH 160/240] Add a basic map/reduce example to show how this can be done Since we can create tasks, run them in a parallel then direct there result into a result task we can create a workflow that can define how this can be accomplished and have the map ops run in parallel (with the reduction op happening after all the map ops have finished). Part of blueprint more-examples Change-Id: I7c04f5508b35b945c49e5798ece0e298d2e1b979 --- doc/source/examples.rst | 12 +++ taskflow/examples/simple_map_reduce.py | 115 +++++++++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 taskflow/examples/simple_map_reduce.py diff --git a/doc/source/examples.rst b/doc/source/examples.rst index 24a43dd0..5043626d 100644 --- a/doc/source/examples.rst +++ b/doc/source/examples.rst @@ -104,6 +104,18 @@ Creating a volume (in parallel) :linenos: :lines: 16- +Summation mapper(s) and reducer (in parallel) +============================================= + +.. note:: + + Full source located at :example:`simple_map_reduce` + +.. literalinclude:: ../../taskflow/examples/simple_map_reduce.py + :language: python + :linenos: + :lines: 16- + Storing & emitting a bill ========================= diff --git a/taskflow/examples/simple_map_reduce.py b/taskflow/examples/simple_map_reduce.py new file mode 100644 index 00000000..3a47fdc1 --- /dev/null +++ b/taskflow/examples/simple_map_reduce.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import sys + +logging.basicConfig(level=logging.ERROR) + +self_dir = os.path.abspath(os.path.dirname(__file__)) +top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, + os.pardir)) +sys.path.insert(0, top_dir) +sys.path.insert(0, self_dir) + +# INTRO: this examples shows a simplistic map/reduce implementation where +# a set of mapper(s) will sum a series of input numbers (in parallel) and +# return there individual summed result. A reducer will then use those +# produced values and perform a final summation and this result will then be +# printed (and verified to ensure the calculation was as expected). + +import six + +from taskflow import engines +from taskflow.patterns import linear_flow +from taskflow.patterns import unordered_flow +from taskflow import task + + +class SumMapper(task.Task): + def execute(self, inputs): + # Sums some set of provided inputs. + return sum(inputs) + + +class TotalReducer(task.Task): + def execute(self, *args, **kwargs): + # Reduces all mapped summed outputs into a single value. + total = 0 + for (k, v) in six.iteritems(kwargs): + # If any other kwargs was passed in, we don't want to use those + # in the calculation of the total... + if k.startswith('reduction_'): + total += v + return total + + +def chunk_iter(chunk_size, upperbound): + """Yields back chunk size pieces from zero to upperbound - 1.""" + chunk = [] + for i in range(0, upperbound): + chunk.append(i) + if len(chunk) == chunk_size: + yield chunk + chunk = [] + + +# Upper bound of numbers to sum for example purposes... +UPPER_BOUND = 10000 + +# How many mappers we want to have. +SPLIT = 10 + +# How big of a chunk we want to give each mapper. +CHUNK_SIZE = UPPER_BOUND // SPLIT + +# This will be the workflow we will compose and run. +w = linear_flow.Flow("root") + +# The mappers will run in parallel. +store = {} +provided = [] +mappers = unordered_flow.Flow('map') +for i, chunk in enumerate(chunk_iter(CHUNK_SIZE, UPPER_BOUND)): + mapper_name = 'mapper_%s' % i + # Give that mapper some information to compute. + store[mapper_name] = chunk + # The reducer uses all of the outputs of the mappers, so it needs + # to be recorded that it needs access to them (under a specific name). + provided.append("reduction_%s" % i) + mappers.add(SumMapper(name=mapper_name, + rebind={'inputs': mapper_name}, + provides=provided[-1])) +w.add(mappers) + +# The reducer will run last (after all the mappers). +w.add(TotalReducer('reducer', requires=provided)) + +# Now go! +e = engines.load(w, engine='parallel', store=store, max_workers=4) +print("Running a parallel engine with options: %s" % e.options) +e.run() + +# Now get the result the reducer created. +total = e.storage.get('reducer') +print("Calculated result = %s" % total) + +# Calculate it manually to verify that it worked... +calc_total = sum(range(0, UPPER_BOUND)) +if calc_total != total: + sys.exit(1) From e5ae74d6bf2afe6fc005e5a45cc8ebcb1703132e Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Thu, 18 Dec 2014 01:28:41 +0000 Subject: [PATCH 161/240] Updated from global requirements Change-Id: I9bbf5d9082e4e6ee9283c7e11ed0b2474a4e070a --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 36857441..96ab9440 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -24,7 +24,7 @@ kazoo>=1.3.1 # PyMySQL or MySQL-python depending on the python version the tests are being # ran in (MySQL-python is currently preferred for 2.x environments, since # it has been used in openstack for the longest). -alembic>=0.6.4 +alembic>=0.7.1 psycopg2 # Docs build jobs need these packages. From 84b387f8bb56d7c03dbe56a84ddbad94bf46262e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 25 Sep 2014 18:27:05 -0700 Subject: [PATCH 162/240] Rework the in-memory backend This avoids storing direct copies of incoming objects and makes sure that we always merge incoming objects (if a saved object already exists) or create a copy of the incoming object if it does not exist when storing. On retrieval we also always return copies instead of returning the data that is stored internally to avoid the problems that can be hard to detect when users (engine or other) modify those source objects. Fixes bug 1365830 Also fixes a retry test case issue that was discovered due to this more easily useable/understandable memory backend changes... Change-Id: I2afdda7beb71e35f7e12d9fd7ccf90b6c5447274 --- taskflow/persistence/backends/impl_memory.py | 223 +++++++++++++------ taskflow/persistence/logbook.py | 57 ++++- taskflow/tests/unit/test_retries.py | 2 +- taskflow/types/failure.py | 2 +- 4 files changed, 211 insertions(+), 73 deletions(-) diff --git a/taskflow/persistence/backends/impl_memory.py b/taskflow/persistence/backends/impl_memory.py index 6c58718a..d266c17f 100644 --- a/taskflow/persistence/backends/impl_memory.py +++ b/taskflow/persistence/backends/impl_memory.py @@ -15,16 +15,97 @@ # License for the specific language governing permissions and limitations # under the License. +import functools + import six from taskflow import exceptions as exc from taskflow import logging from taskflow.persistence.backends import base from taskflow.persistence import logbook +from taskflow.utils import lock_utils LOG = logging.getLogger(__name__) +class _Memory(object): + """Where the data is really stored.""" + + def __init__(self): + self.log_books = {} + self.flow_details = {} + self.atom_details = {} + + def clear_all(self): + self.log_books.clear() + self.flow_details.clear() + self.atom_details.clear() + + +class _MemoryHelper(object): + """Helper functionality for the memory backends & connections.""" + + def __init__(self, memory): + self._memory = memory + + @staticmethod + def _fetch_clone_args(incoming): + if isinstance(incoming, (logbook.LogBook, logbook.FlowDetail)): + # We keep our own copy of the added contents of the following + # types so we don't need the clone to retain them directly... + return { + 'retain_contents': False, + } + return {} + + def construct(self, uuid, container): + """Reconstructs a object from the given uuid and storage container.""" + source = container[uuid] + clone_kwargs = self._fetch_clone_args(source) + clone = source['object'].copy(**clone_kwargs) + rebuilder = source.get('rebuilder') + if rebuilder: + for component in map(rebuilder, source['components']): + clone.add(component) + return clone + + def merge(self, incoming, saved_info=None): + """Merges the incoming object into the local memories copy.""" + if saved_info is None: + if isinstance(incoming, logbook.LogBook): + saved_info = self._memory.log_books.setdefault( + incoming.uuid, {}) + elif isinstance(incoming, logbook.FlowDetail): + saved_info = self._memory.flow_details.setdefault( + incoming.uuid, {}) + elif isinstance(incoming, logbook.AtomDetail): + saved_info = self._memory.atom_details.setdefault( + incoming.uuid, {}) + else: + raise TypeError("Unknown how to merge type '%s'" + % type(incoming)) + try: + saved_info['object'].merge(incoming) + except KeyError: + clone_kwargs = self._fetch_clone_args(incoming) + saved_info['object'] = incoming.copy(**clone_kwargs) + if isinstance(incoming, logbook.LogBook): + flow_details = saved_info.setdefault('components', set()) + if 'rebuilder' not in saved_info: + saved_info['rebuilder'] = functools.partial( + self.construct, container=self._memory.flow_details) + for flow_detail in incoming: + flow_details.add(self.merge(flow_detail)) + elif isinstance(incoming, logbook.FlowDetail): + atom_details = saved_info.setdefault('components', set()) + if 'rebuilder' not in saved_info: + saved_info['rebuilder'] = functools.partial( + self.construct, container=self._memory.atom_details) + for atom_detail in incoming: + atom_details.add(self.merge(atom_detail)) + return incoming.uuid + + class MemoryBackend(base.Backend): """A in-memory (non-persistent) backend. @@ -33,21 +114,28 @@ class MemoryBackend(base.Backend): """ def __init__(self, conf=None): super(MemoryBackend, self).__init__(conf) - self._log_books = {} - self._flow_details = {} - self._atom_details = {} + self._memory = _Memory() + self._helper = _MemoryHelper(self._memory) + self._lock = lock_utils.ReaderWriterLock() + + def _construct_from(self, container): + return dict((uuid, self._helper.construct(uuid, container)) + for uuid in six.iterkeys(container)) @property def log_books(self): - return self._log_books + with self._lock.read_lock(): + return self._construct_from(self._memory.log_books) @property def flow_details(self): - return self._flow_details + with self._lock.read_lock(): + return self._construct_from(self._memory.flow_details) @property def atom_details(self): - return self._atom_details + with self._lock.read_lock(): + return self._construct_from(self._memory.atom_details) def get_connection(self): return Connection(self) @@ -57,8 +145,13 @@ class MemoryBackend(base.Backend): class Connection(base.Connection): + """A connection to an in-memory backend.""" + def __init__(self, backend): self._backend = backend + self._helper = backend._helper + self._memory = backend._memory + self._lock = backend._lock def upgrade(self): pass @@ -74,78 +167,70 @@ class Connection(base.Connection): pass def clear_all(self): - count = 0 - for book_uuid in list(six.iterkeys(self.backend.log_books)): - self.destroy_logbook(book_uuid) - count += 1 - return count + with self._lock.write_lock(): + self._memory.clear_all() def destroy_logbook(self, book_uuid): - try: - # Do the same cascading delete that the sql layer does. - lb = self.backend.log_books.pop(book_uuid) - for fd in lb: - self.backend.flow_details.pop(fd.uuid, None) - for ad in fd: - self.backend.atom_details.pop(ad.uuid, None) - except KeyError: - raise exc.NotFound("No logbook found with id: %s" % book_uuid) + with self._lock.write_lock(): + try: + # Do the same cascading delete that the sql layer does. + book_info = self._memory.log_books.pop(book_uuid) + except KeyError: + raise exc.NotFound("No logbook found with uuid '%s'" + % book_uuid) + else: + while book_info['components']: + flow_uuid = book_info['components'].pop() + flow_info = self._memory.flow_details.pop(flow_uuid) + while flow_info['components']: + atom_uuid = flow_info['components'].pop() + self._memory.atom_details.pop(atom_uuid) def update_atom_details(self, atom_detail): - try: - e_ad = self.backend.atom_details[atom_detail.uuid] - except KeyError: - raise exc.NotFound("No atom details found with id: %s" - % atom_detail.uuid) - return e_ad.merge(atom_detail, deep_copy=True) - - def _save_flowdetail_atoms(self, e_fd, flow_detail): - for atom_detail in flow_detail: - e_ad = e_fd.find(atom_detail.uuid) - if e_ad is None: - e_fd.add(atom_detail) - self.backend.atom_details[atom_detail.uuid] = atom_detail - else: - e_ad.merge(atom_detail, deep_copy=True) + with self._lock.write_lock(): + try: + atom_info = self._memory.atom_details[atom_detail.uuid] + return self._helper.construct( + self._helper.merge(atom_detail, saved_info=atom_info), + self._memory.atom_details) + except KeyError: + raise exc.NotFound("No atom details found with uuid '%s'" + % atom_detail.uuid) def update_flow_details(self, flow_detail): - try: - e_fd = self.backend.flow_details[flow_detail.uuid] - except KeyError: - raise exc.NotFound("No flow details found with id: %s" - % flow_detail.uuid) - e_fd.merge(flow_detail, deep_copy=True) - self._save_flowdetail_atoms(e_fd, flow_detail) - return e_fd + with self._lock.write_lock(): + try: + flow_info = self._memory.flow_details[flow_detail.uuid] + return self._helper.construct( + self._helper.merge(flow_detail, saved_info=flow_info), + self._memory.flow_details) + except KeyError: + raise exc.NotFound("No flow details found with uuid '%s'" + % flow_detail.uuid) def save_logbook(self, book): - # Get a existing logbook model (or create it if it isn't there). - try: - e_lb = self.backend.log_books[book.uuid] - except KeyError: - e_lb = logbook.LogBook(book.name, uuid=book.uuid) - self.backend.log_books[e_lb.uuid] = e_lb - - e_lb.merge(book, deep_copy=True) - # Add anything in to the new logbook that isn't already in the existing - # logbook. - for flow_detail in book: - try: - e_fd = self.backend.flow_details[flow_detail.uuid] - except KeyError: - e_fd = logbook.FlowDetail(flow_detail.name, flow_detail.uuid) - e_lb.add(e_fd) - self.backend.flow_details[e_fd.uuid] = e_fd - e_fd.merge(flow_detail, deep_copy=True) - self._save_flowdetail_atoms(e_fd, flow_detail) - return e_lb + with self._lock.write_lock(): + return self._helper.construct(self._helper.merge(book), + self._memory.log_books) def get_logbook(self, book_uuid): - try: - return self.backend.log_books[book_uuid] - except KeyError: - raise exc.NotFound("No logbook found with id: %s" % book_uuid) + with self._lock.read_lock(): + try: + return self._helper.construct(book_uuid, + self._memory.log_books) + except KeyError: + raise exc.NotFound("No logbook found with uuid '%s'" + % book_uuid) def get_logbooks(self): - for lb in list(six.itervalues(self.backend.log_books)): - yield lb + # Don't hold locks while iterating... + with self._lock.read_lock(): + book_uuids = set(six.iterkeys(self._memory.log_books)) + for book_uuid in book_uuids: + try: + with self._lock.read_lock(): + book = self._helper.construct(book_uuid, + self._memory.log_books) + yield book + except KeyError: + pass diff --git a/taskflow/persistence/logbook.py b/taskflow/persistence/logbook.py index ea6de4d0..f84f37c3 100644 --- a/taskflow/persistence/logbook.py +++ b/taskflow/persistence/logbook.py @@ -137,7 +137,7 @@ class LogBook(object): @classmethod def from_dict(cls, data, unmarshal_time=False): - """Translates the given data into an instance of this class.""" + """Translates the given dictionary into an instance of this class.""" if not unmarshal_time: unmarshal_fn = lambda x: x else: @@ -163,6 +163,17 @@ class LogBook(object): def __len__(self): return len(self._flowdetails_by_id) + def copy(self, retain_contents=True): + """Copies/clones this log book.""" + clone = copy.copy(self) + if not retain_contents: + clone._flowdetails_by_id = {} + else: + clone._flowdetails_by_id = self._flowdetails_by_id.copy() + if self.meta: + clone.meta = self.meta.copy() + return clone + class FlowDetail(object): """A container of atom details, a name and associated metadata. @@ -186,7 +197,7 @@ class FlowDetail(object): """Updates the objects state to be the same as the given one.""" if fd is self: return self - self._atomdetails_by_id = dict(fd._atomdetails_by_id) + self._atomdetails_by_id = fd._atomdetails_by_id self.state = fd.state self.meta = fd.meta return self @@ -206,6 +217,17 @@ class FlowDetail(object): self.state = fd.state return self + def copy(self, retain_contents=True): + """Copies/clones this flow detail.""" + clone = copy.copy(self) + if not retain_contents: + clone._atomdetails_by_id = {} + else: + clone._atomdetails_by_id = self._atomdetails_by_id.copy() + if self.meta: + clone.meta = self.meta.copy() + return clone + def to_dict(self): """Translates the internal state of this object to a dictionary. @@ -380,6 +402,7 @@ class AtomDetail(object): class TaskDetail(AtomDetail): """This class represents a task detail for flow task object.""" + def __init__(self, name, uuid): super(TaskDetail, self).__init__(name, uuid) @@ -410,6 +433,7 @@ class TaskDetail(AtomDetail): return self._to_dict_shared() def merge(self, other, deep_copy=False): + """Merges the current object state with the given ones state.""" if not isinstance(other, TaskDetail): raise exc.NotImplementedError("Can only merge with other" " task details") @@ -421,6 +445,16 @@ class TaskDetail(AtomDetail): self.results = copy_fn(other.results) return self + def copy(self): + """Copies/clones this task detail.""" + clone = copy.copy(self) + clone.results = copy.copy(self.results) + if self.meta: + clone.meta = self.meta.copy() + if self.version: + clone.version = copy.copy(self.version) + return clone + class RetryDetail(AtomDetail): """This class represents a retry detail for retry controller object.""" @@ -434,6 +468,24 @@ class RetryDetail(AtomDetail): self.state = state self.intention = states.EXECUTE + def copy(self): + """Copies/clones this retry detail.""" + clone = copy.copy(self) + results = [] + # NOTE(imelnikov): we can't just deep copy Failures, as they + # contain tracebacks, which are not copyable. + for (data, failures) in self.results: + copied_failures = {} + for (key, failure) in six.iteritems(failures): + copied_failures[key] = failure + results.append((data, copied_failures)) + clone.results = results + if self.meta: + clone.meta = self.meta.copy() + if self.version: + clone.version = copy.copy(self.version) + return clone + @property def last_results(self): try: @@ -496,6 +548,7 @@ class RetryDetail(AtomDetail): return base def merge(self, other, deep_copy=False): + """Merges the current object state with the given ones state.""" if not isinstance(other, RetryDetail): raise exc.NotImplementedError("Can only merge with other" " retry details") diff --git a/taskflow/tests/unit/test_retries.py b/taskflow/tests/unit/test_retries.py index 54400435..09532010 100644 --- a/taskflow/tests/unit/test_retries.py +++ b/taskflow/tests/unit/test_retries.py @@ -559,7 +559,7 @@ class RetryTest(utils.EngineTestBase): # we execute retry engine.storage.save('flow-1_retry', 1) # task fails - fail = failure.Failure.from_exception(RuntimeError('foo')), + fail = failure.Failure.from_exception(RuntimeError('foo')) engine.storage.save('task1', fail, state=st.FAILURE) if when == 'task fails': return engine diff --git a/taskflow/types/failure.py b/taskflow/types/failure.py index b9d7a399..87ff1bf6 100644 --- a/taskflow/types/failure.py +++ b/taskflow/types/failure.py @@ -74,7 +74,7 @@ def _are_equal_exc_info_tuples(ei1, ei2): class Failure(object): - """Object that represents failure. + """An immutable object that represents failure. Failure objects encapsulate exception information so that they can be re-used later to re-raise, inspect, examine, log, print, serialize, From 4e514f41e57983e728db9025126df6f791a2594a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 8 Dec 2014 18:58:18 -0800 Subject: [PATCH 163/240] Move over to using oslo.utils [reflection, uuidutils] The reflection module is now part of oslo.utils so we should remove our local version and use that version instead; this also goes for the uuidutils module which is now part of oslo.utils as well so we no longer need our local version copied from the incubator... Note that one reflection method `find_subclasses` which was to specific to taskflow is now moved to the misc utility module instead of its prior home in the reflection module. Change-Id: I069881c80b0b2916cc0c414992b80171f7eeb79f --- doc/source/utils.rst | 5 - openstack-common.conf | 1 - taskflow/atom.py | 2 +- taskflow/engines/action_engine/engine.py | 2 +- taskflow/engines/helpers.py | 2 +- taskflow/engines/worker_based/endpoint.py | 3 +- taskflow/engines/worker_based/executor.py | 2 +- taskflow/engines/worker_based/protocol.py | 2 +- taskflow/engines/worker_based/worker.py | 5 +- taskflow/examples/create_parallel_volume.py | 3 +- taskflow/examples/fake_billing.py | 2 +- taskflow/examples/resume_vm_boot.py | 3 +- taskflow/flow.py | 3 +- taskflow/jobs/backends/impl_zookeeper.py | 2 +- taskflow/jobs/job.py | 3 +- taskflow/openstack/__init__.py | 0 taskflow/openstack/common/__init__.py | 17 -- taskflow/openstack/common/uuidutils.py | 37 --- .../persistence/backends/sqlalchemy/models.py | 2 +- taskflow/persistence/logbook.py | 2 +- taskflow/storage.py | 4 +- taskflow/task.py | 2 +- taskflow/tests/unit/jobs/base.py | 2 +- taskflow/tests/unit/jobs/test_zk_job.py | 2 +- taskflow/tests/unit/persistence/base.py | 3 +- .../unit/persistence/test_zk_persistence.py | 2 +- taskflow/tests/unit/test_listeners.py | 2 +- taskflow/tests/unit/test_storage.py | 3 +- taskflow/tests/unit/test_utils.py | 255 ------------------ .../tests/unit/worker_based/test_endpoint.py | 3 +- .../unit/worker_based/test_message_pump.py | 3 +- .../tests/unit/worker_based/test_pipeline.py | 2 +- .../tests/unit/worker_based/test_protocol.py | 2 +- .../tests/unit/worker_based/test_worker.py | 2 +- taskflow/types/cache.py | 3 +- taskflow/types/failure.py | 2 +- taskflow/types/notifier.py | 3 +- taskflow/utils/deprecation.py | 3 +- taskflow/utils/kazoo_utils.py | 2 +- taskflow/utils/misc.py | 48 +++- taskflow/utils/persistence_utils.py | 2 +- taskflow/utils/reflection.py | 251 ----------------- 42 files changed, 91 insertions(+), 608 deletions(-) delete mode 100644 taskflow/openstack/__init__.py delete mode 100644 taskflow/openstack/common/__init__.py delete mode 100644 taskflow/openstack/common/uuidutils.py delete mode 100644 taskflow/utils/reflection.py diff --git a/doc/source/utils.rst b/doc/source/utils.rst index 968c2c04..8125aac6 100644 --- a/doc/source/utils.rst +++ b/doc/source/utils.rst @@ -38,11 +38,6 @@ Persistence .. automodule:: taskflow.utils.persistence_utils -Reflection -~~~~~~~~~~ - -.. automodule:: taskflow.utils.reflection - Threading ~~~~~~~~~ diff --git a/openstack-common.conf b/openstack-common.conf index 9db6be0a..127bc839 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,7 +1,6 @@ [DEFAULT] # The list of modules to copy from oslo-incubator.git -module=uuidutils script=tools/run_cross_tests.sh # The base module to hold the copy of openstack.common diff --git a/taskflow/atom.py b/taskflow/atom.py index 2cd665ac..3131664f 100644 --- a/taskflow/atom.py +++ b/taskflow/atom.py @@ -15,11 +15,11 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.utils import reflection import six from taskflow import exceptions from taskflow.utils import misc -from taskflow.utils import reflection def _save_as_to_mapping(save_as): diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index ffc3a80a..0f2a8a20 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -18,6 +18,7 @@ import contextlib import threading from oslo.utils import excutils +from oslo.utils import reflection from taskflow.engines.action_engine import compiler from taskflow.engines.action_engine import executor @@ -29,7 +30,6 @@ from taskflow import storage as atom_storage from taskflow.types import failure from taskflow.utils import lock_utils from taskflow.utils import misc -from taskflow.utils import reflection @contextlib.contextmanager diff --git a/taskflow/engines/helpers.py b/taskflow/engines/helpers.py index 1943aa73..2c84a3c3 100644 --- a/taskflow/engines/helpers.py +++ b/taskflow/engines/helpers.py @@ -19,6 +19,7 @@ import itertools import traceback from oslo.utils import importutils +from oslo.utils import reflection import six import stevedore.driver @@ -28,7 +29,6 @@ from taskflow.persistence import backends as p_backends from taskflow.utils import deprecation from taskflow.utils import misc from taskflow.utils import persistence_utils as p_utils -from taskflow.utils import reflection LOG = logging.getLogger(__name__) diff --git a/taskflow/engines/worker_based/endpoint.py b/taskflow/engines/worker_based/endpoint.py index 58637e11..0f883ade 100644 --- a/taskflow/engines/worker_based/endpoint.py +++ b/taskflow/engines/worker_based/endpoint.py @@ -14,8 +14,9 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.utils import reflection + from taskflow.engines.action_engine import executor -from taskflow.utils import reflection class Endpoint(object): diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index bdef7bff..450f493f 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -17,6 +17,7 @@ import functools import threading +from oslo.utils import reflection from oslo.utils import timeutils from taskflow.engines.action_engine import executor @@ -28,7 +29,6 @@ from taskflow import logging from taskflow import task as task_atom from taskflow.types import timing as tt from taskflow.utils import misc -from taskflow.utils import reflection from taskflow.utils import threading_utils as tu LOG = logging.getLogger(__name__) diff --git a/taskflow/engines/worker_based/protocol.py b/taskflow/engines/worker_based/protocol.py index e2b40e60..96ba84c4 100644 --- a/taskflow/engines/worker_based/protocol.py +++ b/taskflow/engines/worker_based/protocol.py @@ -20,6 +20,7 @@ import threading from concurrent import futures import jsonschema from jsonschema import exceptions as schema_exc +from oslo.utils import reflection from oslo.utils import timeutils import six @@ -29,7 +30,6 @@ from taskflow import logging from taskflow.types import failure as ft from taskflow.types import timing as tt from taskflow.utils import lock_utils -from taskflow.utils import reflection # NOTE(skudriashev): This is protocol states and events, which are not # related to task states. diff --git a/taskflow/engines/worker_based/worker.py b/taskflow/engines/worker_based/worker.py index 18627e27..f75b7a8d 100644 --- a/taskflow/engines/worker_based/worker.py +++ b/taskflow/engines/worker_based/worker.py @@ -21,12 +21,13 @@ import string import sys from concurrent import futures +from oslo.utils import reflection from taskflow.engines.worker_based import endpoint from taskflow.engines.worker_based import server from taskflow import logging from taskflow import task as t_task -from taskflow.utils import reflection +from taskflow.utils import misc from taskflow.utils import threading_utils as tu from taskflow import version @@ -103,7 +104,7 @@ class Worker(object): @staticmethod def _derive_endpoints(tasks): """Derive endpoints from list of strings, classes or packages.""" - derived_tasks = reflection.find_subclasses(tasks, t_task.BaseTask) + derived_tasks = misc.find_subclasses(tasks, t_task.BaseTask) return [endpoint.Endpoint(task) for task in derived_tasks] def _generate_banner(self): diff --git a/taskflow/examples/create_parallel_volume.py b/taskflow/examples/create_parallel_volume.py index 5185330b..0416a25e 100644 --- a/taskflow/examples/create_parallel_volume.py +++ b/taskflow/examples/create_parallel_volume.py @@ -28,11 +28,12 @@ top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) sys.path.insert(0, top_dir) +from oslo.utils import reflection + from taskflow import engines from taskflow.listeners import printing from taskflow.patterns import unordered_flow as uf from taskflow import task -from taskflow.utils import reflection # INTRO: This examples shows how unordered_flow can be used to create a large # number of fake volumes in parallel (or serially, depending on a constant that diff --git a/taskflow/examples/fake_billing.py b/taskflow/examples/fake_billing.py index 22c75cd9..0fbe81f7 100644 --- a/taskflow/examples/fake_billing.py +++ b/taskflow/examples/fake_billing.py @@ -27,10 +27,10 @@ top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) sys.path.insert(0, top_dir) +from oslo.utils import uuidutils from taskflow import engines from taskflow.listeners import printing -from taskflow.openstack.common import uuidutils from taskflow.patterns import graph_flow as gf from taskflow.patterns import linear_flow as lf from taskflow import task diff --git a/taskflow/examples/resume_vm_boot.py b/taskflow/examples/resume_vm_boot.py index 514f3336..f400d0d1 100644 --- a/taskflow/examples/resume_vm_boot.py +++ b/taskflow/examples/resume_vm_boot.py @@ -31,9 +31,10 @@ top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), sys.path.insert(0, top_dir) sys.path.insert(0, self_dir) +from oslo.utils import uuidutils + from taskflow import engines from taskflow import exceptions as exc -from taskflow.openstack.common import uuidutils from taskflow.patterns import graph_flow as gf from taskflow.patterns import linear_flow as lf from taskflow import task diff --git a/taskflow/flow.py b/taskflow/flow.py index c5fb4296..683cf8b4 100644 --- a/taskflow/flow.py +++ b/taskflow/flow.py @@ -16,10 +16,9 @@ import abc +from oslo.utils import reflection import six -from taskflow.utils import reflection - # Link metadata keys that have inherent/special meaning. # # This key denotes the link is an invariant that ensures the order is diff --git a/taskflow/jobs/backends/impl_zookeeper.py b/taskflow/jobs/backends/impl_zookeeper.py index e25ac51d..65f88902 100644 --- a/taskflow/jobs/backends/impl_zookeeper.py +++ b/taskflow/jobs/backends/impl_zookeeper.py @@ -25,13 +25,13 @@ from kazoo.protocol import paths as k_paths from kazoo.recipe import watchers from oslo.serialization import jsonutils from oslo.utils import excutils +from oslo.utils import uuidutils import six from taskflow import exceptions as excp from taskflow.jobs import job as base_job from taskflow.jobs import jobboard from taskflow import logging -from taskflow.openstack.common import uuidutils from taskflow import states from taskflow.types import timing as tt from taskflow.utils import kazoo_utils diff --git a/taskflow/jobs/job.py b/taskflow/jobs/job.py index 41ac4c16..23e33ee7 100644 --- a/taskflow/jobs/job.py +++ b/taskflow/jobs/job.py @@ -17,10 +17,9 @@ import abc +from oslo.utils import uuidutils import six -from taskflow.openstack.common import uuidutils - @six.add_metaclass(abc.ABCMeta) class Job(object): diff --git a/taskflow/openstack/__init__.py b/taskflow/openstack/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/taskflow/openstack/common/__init__.py b/taskflow/openstack/common/__init__.py deleted file mode 100644 index d1223eaf..00000000 --- a/taskflow/openstack/common/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import six - - -six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox')) diff --git a/taskflow/openstack/common/uuidutils.py b/taskflow/openstack/common/uuidutils.py deleted file mode 100644 index 234b880c..00000000 --- a/taskflow/openstack/common/uuidutils.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2012 Intel Corporation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -UUID related utilities and helper functions. -""" - -import uuid - - -def generate_uuid(): - return str(uuid.uuid4()) - - -def is_uuid_like(val): - """Returns validation of a value as a UUID. - - For our purposes, a UUID is a canonical form string: - aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa - - """ - try: - return str(uuid.UUID(val)) == val - except (TypeError, ValueError, AttributeError): - return False diff --git a/taskflow/persistence/backends/sqlalchemy/models.py b/taskflow/persistence/backends/sqlalchemy/models.py index 47b8c839..3f056de5 100644 --- a/taskflow/persistence/backends/sqlalchemy/models.py +++ b/taskflow/persistence/backends/sqlalchemy/models.py @@ -17,6 +17,7 @@ from oslo.serialization import jsonutils from oslo.utils import timeutils +from oslo.utils import uuidutils from sqlalchemy import Column, String, DateTime, Enum from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import ForeignKey @@ -24,7 +25,6 @@ from sqlalchemy.orm import backref from sqlalchemy.orm import relationship from sqlalchemy import types as types -from taskflow.openstack.common import uuidutils from taskflow.persistence import logbook from taskflow import states diff --git a/taskflow/persistence/logbook.py b/taskflow/persistence/logbook.py index ea6de4d0..6ae7d7c9 100644 --- a/taskflow/persistence/logbook.py +++ b/taskflow/persistence/logbook.py @@ -19,11 +19,11 @@ import abc import copy from oslo.utils import timeutils +from oslo.utils import uuidutils import six from taskflow import exceptions as exc from taskflow import logging -from taskflow.openstack.common import uuidutils from taskflow import states from taskflow.types import failure as ft diff --git a/taskflow/storage.py b/taskflow/storage.py index dcc51714..e698ca4d 100644 --- a/taskflow/storage.py +++ b/taskflow/storage.py @@ -17,11 +17,12 @@ import abc import contextlib +from oslo.utils import reflection +from oslo.utils import uuidutils import six from taskflow import exceptions from taskflow import logging -from taskflow.openstack.common import uuidutils from taskflow.persistence import logbook from taskflow import retry from taskflow import states @@ -29,7 +30,6 @@ from taskflow import task from taskflow.types import failure from taskflow.utils import lock_utils from taskflow.utils import misc -from taskflow.utils import reflection LOG = logging.getLogger(__name__) STATES_WITH_RESULTS = (states.SUCCESS, states.REVERTING, states.FAILURE) diff --git a/taskflow/task.py b/taskflow/task.py index fbee0296..7a1c7180 100644 --- a/taskflow/task.py +++ b/taskflow/task.py @@ -18,13 +18,13 @@ import abc import copy +from oslo.utils import reflection import six from taskflow import atom from taskflow import logging from taskflow.types import notifier from taskflow.utils import misc -from taskflow.utils import reflection LOG = logging.getLogger(__name__) diff --git a/taskflow/tests/unit/jobs/base.py b/taskflow/tests/unit/jobs/base.py index 24da7d3a..4c070f31 100644 --- a/taskflow/tests/unit/jobs/base.py +++ b/taskflow/tests/unit/jobs/base.py @@ -18,9 +18,9 @@ import contextlib import time from kazoo.recipe import watchers +from oslo.utils import uuidutils from taskflow import exceptions as excp -from taskflow.openstack.common import uuidutils from taskflow.persistence.backends import impl_dir from taskflow import states from taskflow.test import mock diff --git a/taskflow/tests/unit/jobs/test_zk_job.py b/taskflow/tests/unit/jobs/test_zk_job.py index 5a536f9e..8737a4f5 100644 --- a/taskflow/tests/unit/jobs/test_zk_job.py +++ b/taskflow/tests/unit/jobs/test_zk_job.py @@ -15,13 +15,13 @@ # under the License. from oslo.serialization import jsonutils +from oslo.utils import uuidutils import six import testtools from zake import fake_client from zake import utils as zake_utils from taskflow.jobs.backends import impl_zookeeper -from taskflow.openstack.common import uuidutils from taskflow import states from taskflow import test from taskflow.tests.unit.jobs import base diff --git a/taskflow/tests/unit/persistence/base.py b/taskflow/tests/unit/persistence/base.py index 50bb3b3f..88660fd5 100644 --- a/taskflow/tests/unit/persistence/base.py +++ b/taskflow/tests/unit/persistence/base.py @@ -16,8 +16,9 @@ import contextlib +from oslo.utils import uuidutils + from taskflow import exceptions as exc -from taskflow.openstack.common import uuidutils from taskflow.persistence import logbook from taskflow import states from taskflow.types import failure diff --git a/taskflow/tests/unit/persistence/test_zk_persistence.py b/taskflow/tests/unit/persistence/test_zk_persistence.py index 609de21f..28463bb7 100644 --- a/taskflow/tests/unit/persistence/test_zk_persistence.py +++ b/taskflow/tests/unit/persistence/test_zk_persistence.py @@ -17,11 +17,11 @@ import contextlib from kazoo import exceptions as kazoo_exceptions +from oslo.utils import uuidutils import testtools from zake import fake_client from taskflow import exceptions as exc -from taskflow.openstack.common import uuidutils from taskflow.persistence import backends from taskflow.persistence.backends import impl_zookeeper from taskflow import test diff --git a/taskflow/tests/unit/test_listeners.py b/taskflow/tests/unit/test_listeners.py index 210fe798..c10bc282 100644 --- a/taskflow/tests/unit/test_listeners.py +++ b/taskflow/tests/unit/test_listeners.py @@ -19,6 +19,7 @@ import logging import time from oslo.serialization import jsonutils +from oslo.utils import reflection import six from zake import fake_client @@ -37,7 +38,6 @@ from taskflow.test import mock from taskflow.tests import utils as test_utils from taskflow.utils import misc from taskflow.utils import persistence_utils -from taskflow.utils import reflection from taskflow.utils import threading_utils diff --git a/taskflow/tests/unit/test_storage.py b/taskflow/tests/unit/test_storage.py index f774993c..886e075a 100644 --- a/taskflow/tests/unit/test_storage.py +++ b/taskflow/tests/unit/test_storage.py @@ -17,8 +17,9 @@ import contextlib import threading +from oslo.utils import uuidutils + from taskflow import exceptions -from taskflow.openstack.common import uuidutils from taskflow.persistence import backends from taskflow.persistence import logbook from taskflow import states diff --git a/taskflow/tests/unit/test_utils.py b/taskflow/tests/unit/test_utils.py index 56e39199..c69b769d 100644 --- a/taskflow/tests/unit/test_utils.py +++ b/taskflow/tests/unit/test_utils.py @@ -19,266 +19,11 @@ import inspect import random import time -import six -import testtools - from taskflow import test -from taskflow.tests import utils as test_utils -from taskflow.types import failure -from taskflow.utils import lock_utils from taskflow.utils import misc -from taskflow.utils import reflection from taskflow.utils import threading_utils -def mere_function(a, b): - pass - - -def function_with_defs(a, b, optional=None): - pass - - -def function_with_kwargs(a, b, **kwargs): - pass - - -class Class(object): - - def method(self, c, d): - pass - - @staticmethod - def static_method(e, f): - pass - - @classmethod - def class_method(cls, g, h): - pass - - -class CallableClass(object): - def __call__(self, i, j): - pass - - -class ClassWithInit(object): - def __init__(self, k, l): - pass - - -class CallbackEqualityTest(test.TestCase): - def test_different_simple_callbacks(self): - - def a(): - pass - - def b(): - pass - - self.assertFalse(reflection.is_same_callback(a, b)) - - def test_static_instance_callbacks(self): - - class A(object): - - @staticmethod - def b(a, b, c): - pass - - a = A() - b = A() - - self.assertTrue(reflection.is_same_callback(a.b, b.b)) - - def test_different_instance_callbacks(self): - - class A(object): - def b(self): - pass - - def __eq__(self, other): - return True - - b = A() - c = A() - - self.assertFalse(reflection.is_same_callback(b.b, c.b)) - self.assertTrue(reflection.is_same_callback(b.b, c.b, strict=False)) - - -class GetCallableNameTest(test.TestCase): - - def test_mere_function(self): - name = reflection.get_callable_name(mere_function) - self.assertEqual(name, '.'.join((__name__, 'mere_function'))) - - def test_method(self): - name = reflection.get_callable_name(Class.method) - self.assertEqual(name, '.'.join((__name__, 'Class', 'method'))) - - def test_instance_method(self): - name = reflection.get_callable_name(Class().method) - self.assertEqual(name, '.'.join((__name__, 'Class', 'method'))) - - def test_static_method(self): - name = reflection.get_callable_name(Class.static_method) - if six.PY3: - self.assertEqual(name, - '.'.join((__name__, 'Class', 'static_method'))) - else: - # NOTE(imelnikov): static method are just functions, class name - # is not recorded anywhere in them. - self.assertEqual(name, - '.'.join((__name__, 'static_method'))) - - def test_class_method(self): - name = reflection.get_callable_name(Class.class_method) - self.assertEqual(name, '.'.join((__name__, 'Class', 'class_method'))) - - def test_constructor(self): - name = reflection.get_callable_name(Class) - self.assertEqual(name, '.'.join((__name__, 'Class'))) - - def test_callable_class(self): - name = reflection.get_callable_name(CallableClass()) - self.assertEqual(name, '.'.join((__name__, 'CallableClass'))) - - def test_callable_class_call(self): - name = reflection.get_callable_name(CallableClass().__call__) - self.assertEqual(name, '.'.join((__name__, 'CallableClass', - '__call__'))) - - -# These extended/special case tests only work on python 3, due to python 2 -# being broken/incorrect with regard to these special cases... -@testtools.skipIf(not six.PY3, 'python 3.x is not currently available') -class GetCallableNameTestExtended(test.TestCase): - # Tests items in http://legacy.python.org/dev/peps/pep-3155/ - - class InnerCallableClass(object): - def __call__(self): - pass - - def test_inner_callable_class(self): - obj = self.InnerCallableClass() - name = reflection.get_callable_name(obj.__call__) - expected_name = '.'.join((__name__, 'GetCallableNameTestExtended', - 'InnerCallableClass', '__call__')) - self.assertEqual(expected_name, name) - - def test_inner_callable_function(self): - def a(): - - def b(): - pass - - return b - - name = reflection.get_callable_name(a()) - expected_name = '.'.join((__name__, 'GetCallableNameTestExtended', - 'test_inner_callable_function', '', - 'a', '', 'b')) - self.assertEqual(expected_name, name) - - def test_inner_class(self): - obj = self.InnerCallableClass() - name = reflection.get_callable_name(obj) - expected_name = '.'.join((__name__, - 'GetCallableNameTestExtended', - 'InnerCallableClass')) - self.assertEqual(expected_name, name) - - -class GetCallableArgsTest(test.TestCase): - - def test_mere_function(self): - result = reflection.get_callable_args(mere_function) - self.assertEqual(['a', 'b'], result) - - def test_function_with_defaults(self): - result = reflection.get_callable_args(function_with_defs) - self.assertEqual(['a', 'b', 'optional'], result) - - def test_required_only(self): - result = reflection.get_callable_args(function_with_defs, - required_only=True) - self.assertEqual(['a', 'b'], result) - - def test_method(self): - result = reflection.get_callable_args(Class.method) - self.assertEqual(['self', 'c', 'd'], result) - - def test_instance_method(self): - result = reflection.get_callable_args(Class().method) - self.assertEqual(['c', 'd'], result) - - def test_class_method(self): - result = reflection.get_callable_args(Class.class_method) - self.assertEqual(['g', 'h'], result) - - def test_class_constructor(self): - result = reflection.get_callable_args(ClassWithInit) - self.assertEqual(['k', 'l'], result) - - def test_class_with_call(self): - result = reflection.get_callable_args(CallableClass()) - self.assertEqual(['i', 'j'], result) - - def test_decorators_work(self): - @lock_utils.locked - def locked_fun(x, y): - pass - result = reflection.get_callable_args(locked_fun) - self.assertEqual(['x', 'y'], result) - - -class AcceptsKwargsTest(test.TestCase): - - def test_no_kwargs(self): - self.assertEqual( - reflection.accepts_kwargs(mere_function), False) - - def test_with_kwargs(self): - self.assertEqual( - reflection.accepts_kwargs(function_with_kwargs), True) - - -class GetClassNameTest(test.TestCase): - - def test_std_exception(self): - name = reflection.get_class_name(RuntimeError) - self.assertEqual(name, 'RuntimeError') - - def test_global_class(self): - name = reflection.get_class_name(failure.Failure) - self.assertEqual(name, 'taskflow.types.failure.Failure') - - def test_class(self): - name = reflection.get_class_name(Class) - self.assertEqual(name, '.'.join((__name__, 'Class'))) - - def test_instance(self): - name = reflection.get_class_name(Class()) - self.assertEqual(name, '.'.join((__name__, 'Class'))) - - def test_int(self): - name = reflection.get_class_name(42) - self.assertEqual(name, 'int') - - -class GetAllClassNamesTest(test.TestCase): - - def test_std_class(self): - names = list(reflection.get_all_class_names(RuntimeError)) - self.assertEqual(names, test_utils.RUNTIME_ERROR_CLASSES) - - def test_std_class_up_to(self): - names = list(reflection.get_all_class_names(RuntimeError, - up_to=Exception)) - self.assertEqual(names, test_utils.RUNTIME_ERROR_CLASSES[:-2]) - - class CachedPropertyTest(test.TestCase): def test_attribute_caching(self): diff --git a/taskflow/tests/unit/worker_based/test_endpoint.py b/taskflow/tests/unit/worker_based/test_endpoint.py index 36abb980..53260b12 100644 --- a/taskflow/tests/unit/worker_based/test_endpoint.py +++ b/taskflow/tests/unit/worker_based/test_endpoint.py @@ -14,11 +14,12 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.utils import reflection + from taskflow.engines.worker_based import endpoint as ep from taskflow import task from taskflow import test from taskflow.tests import utils -from taskflow.utils import reflection class Task(task.Task): diff --git a/taskflow/tests/unit/worker_based/test_message_pump.py b/taskflow/tests/unit/worker_based/test_message_pump.py index cae4fa5c..7b945a26 100644 --- a/taskflow/tests/unit/worker_based/test_message_pump.py +++ b/taskflow/tests/unit/worker_based/test_message_pump.py @@ -14,9 +14,10 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.utils import uuidutils + from taskflow.engines.worker_based import protocol as pr from taskflow.engines.worker_based import proxy -from taskflow.openstack.common import uuidutils from taskflow import test from taskflow.test import mock from taskflow.tests import utils as test_utils diff --git a/taskflow/tests/unit/worker_based/test_pipeline.py b/taskflow/tests/unit/worker_based/test_pipeline.py index 2822a852..53bf8f9b 100644 --- a/taskflow/tests/unit/worker_based/test_pipeline.py +++ b/taskflow/tests/unit/worker_based/test_pipeline.py @@ -15,12 +15,12 @@ # under the License. from concurrent import futures +from oslo.utils import uuidutils from taskflow.engines.action_engine import executor as base_executor from taskflow.engines.worker_based import endpoint from taskflow.engines.worker_based import executor as worker_executor from taskflow.engines.worker_based import server as worker_server -from taskflow.openstack.common import uuidutils from taskflow import test from taskflow.tests import utils as test_utils from taskflow.types import failure diff --git a/taskflow/tests/unit/worker_based/test_protocol.py b/taskflow/tests/unit/worker_based/test_protocol.py index d2f2cc02..4c34ed60 100644 --- a/taskflow/tests/unit/worker_based/test_protocol.py +++ b/taskflow/tests/unit/worker_based/test_protocol.py @@ -16,11 +16,11 @@ from concurrent import futures from oslo.utils import timeutils +from oslo.utils import uuidutils from taskflow.engines.action_engine import executor from taskflow.engines.worker_based import protocol as pr from taskflow import exceptions as excp -from taskflow.openstack.common import uuidutils from taskflow import test from taskflow.tests import utils from taskflow.types import failure diff --git a/taskflow/tests/unit/worker_based/test_worker.py b/taskflow/tests/unit/worker_based/test_worker.py index ff049a64..8fc76eb4 100644 --- a/taskflow/tests/unit/worker_based/test_worker.py +++ b/taskflow/tests/unit/worker_based/test_worker.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.utils import reflection import six from taskflow.engines.worker_based import endpoint @@ -21,7 +22,6 @@ from taskflow.engines.worker_based import worker from taskflow import test from taskflow.test import mock from taskflow.tests import utils -from taskflow.utils import reflection class TestWorker(test.MockTestCase): diff --git a/taskflow/types/cache.py b/taskflow/types/cache.py index 61511e12..c3ac7a18 100644 --- a/taskflow/types/cache.py +++ b/taskflow/types/cache.py @@ -16,10 +16,9 @@ import threading +from oslo.utils import reflection import six -from taskflow.utils import reflection - class ExpiringCache(object): """Represents a thread-safe time-based expiring cache. diff --git a/taskflow/types/failure.py b/taskflow/types/failure.py index b9d7a399..33fb345d 100644 --- a/taskflow/types/failure.py +++ b/taskflow/types/failure.py @@ -18,10 +18,10 @@ import copy import sys import traceback +from oslo.utils import reflection import six from taskflow import exceptions as exc -from taskflow.utils import reflection def _copy_exc_info(exc_info): diff --git a/taskflow/types/notifier.py b/taskflow/types/notifier.py index 98511fba..838585a2 100644 --- a/taskflow/types/notifier.py +++ b/taskflow/types/notifier.py @@ -19,10 +19,9 @@ import contextlib import copy import logging +from oslo.utils import reflection import six -from taskflow.utils import reflection - LOG = logging.getLogger(__name__) diff --git a/taskflow/utils/deprecation.py b/taskflow/utils/deprecation.py index f70d7670..5c466662 100644 --- a/taskflow/utils/deprecation.py +++ b/taskflow/utils/deprecation.py @@ -17,10 +17,9 @@ import functools import warnings +from oslo.utils import reflection import six -from taskflow.utils import reflection - _CLASS_MOVED_PREFIX_TPL = "Class '%s' has moved to '%s'" _KIND_MOVED_PREFIX_TPL = "%s '%s' has moved to '%s'" _KWARG_MOVED_POSTFIX_TPL = ", please use the '%s' argument instead" diff --git a/taskflow/utils/kazoo_utils.py b/taskflow/utils/kazoo_utils.py index 0a9922bb..ab449635 100644 --- a/taskflow/utils/kazoo_utils.py +++ b/taskflow/utils/kazoo_utils.py @@ -16,11 +16,11 @@ from kazoo import client from kazoo import exceptions as k_exc +from oslo.utils import reflection import six from six.moves import zip as compat_zip from taskflow import exceptions as exc -from taskflow.utils import reflection def _parse_hosts(hosts): diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 34910064..a6b04fa7 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -23,9 +23,12 @@ import os import re import sys import threading +import types from oslo.serialization import jsonutils +from oslo.utils import importutils from oslo.utils import netutils +from oslo.utils import reflection import six from six.moves import map as compat_map from six.moves import range as compat_range @@ -34,7 +37,6 @@ from six.moves.urllib import parse as urlparse from taskflow.types import failure from taskflow.types import notifier from taskflow.utils import deprecation -from taskflow.utils import reflection NUMERIC_TYPES = six.integer_types + (float,) @@ -83,6 +85,50 @@ def merge_uri(uri, conf): return conf +def find_subclasses(locations, base_cls, exclude_hidden=True): + """Finds subclass types in the given locations. + + This will examines the given locations for types which are subclasses of + the base class type provided and returns the found subclasses (or fails + with exceptions if this introspection can not be accomplished). + + If a string is provided as one of the locations it will be imported and + examined if it is a subclass of the base class. If a module is given, + all of its members will be examined for attributes which are subclasses of + the base class. If a type itself is given it will be examined for being a + subclass of the base class. + """ + derived = set() + for item in locations: + module = None + if isinstance(item, six.string_types): + try: + pkg, cls = item.split(':') + except ValueError: + module = importutils.import_module(item) + else: + obj = importutils.import_class('%s.%s' % (pkg, cls)) + if not reflection.is_subclass(obj, base_cls): + raise TypeError("Item %s is not a %s subclass" % + (item, base_cls)) + derived.add(obj) + elif isinstance(item, types.ModuleType): + module = item + elif reflection.is_subclass(item, base_cls): + derived.add(item) + else: + raise TypeError("Item %s unexpected type: %s" % + (item, type(item))) + # If it's a module derive objects from it if we can. + if module is not None: + for (name, obj) in inspect.getmembers(module): + if name.startswith("_") and exclude_hidden: + continue + if reflection.is_subclass(obj, base_cls): + derived.add(obj) + return derived + + def parse_uri(uri): """Parses a uri into its components.""" # Do some basic validation before continuing... diff --git a/taskflow/utils/persistence_utils.py b/taskflow/utils/persistence_utils.py index 340f558a..b8a15351 100644 --- a/taskflow/utils/persistence_utils.py +++ b/taskflow/utils/persistence_utils.py @@ -17,9 +17,9 @@ import contextlib from oslo.utils import timeutils +from oslo.utils import uuidutils from taskflow import logging -from taskflow.openstack.common import uuidutils from taskflow.persistence import logbook from taskflow.utils import misc diff --git a/taskflow/utils/reflection.py b/taskflow/utils/reflection.py deleted file mode 100644 index 08eaf6c9..00000000 --- a/taskflow/utils/reflection.py +++ /dev/null @@ -1,251 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import inspect -import types - -from oslo.utils import importutils -import six - -try: - _TYPE_TYPE = types.TypeType -except AttributeError: - _TYPE_TYPE = type - -# See: https://docs.python.org/2/library/__builtin__.html#module-__builtin__ -# and see https://docs.python.org/2/reference/executionmodel.html (and likely -# others)... -_BUILTIN_MODULES = ('builtins', '__builtin__', 'exceptions') - - -def _get_members(obj, exclude_hidden): - """Yields the members of an object, filtering by hidden/not hidden.""" - for (name, value) in inspect.getmembers(obj): - if name.startswith("_") and exclude_hidden: - continue - yield (name, value) - - -def find_subclasses(locations, base_cls, exclude_hidden=True): - """Finds subclass types in the given locations. - - This will examines the given locations for types which are subclasses of - the base class type provided and returns the found subclasses (or fails - with exceptions if this introspection can not be accomplished). - - If a string is provided as one of the locations it will be imported and - examined if it is a subclass of the base class. If a module is given, - all of its members will be examined for attributes which are subclasses of - the base class. If a type itself is given it will be examined for being a - subclass of the base class. - """ - derived = set() - for item in locations: - module = None - if isinstance(item, six.string_types): - try: - pkg, cls = item.split(':') - except ValueError: - module = importutils.import_module(item) - else: - obj = importutils.import_class('%s.%s' % (pkg, cls)) - if not is_subclass(obj, base_cls): - raise TypeError("Item %s is not a %s subclass" % - (item, base_cls)) - derived.add(obj) - elif isinstance(item, types.ModuleType): - module = item - elif is_subclass(item, base_cls): - derived.add(item) - else: - raise TypeError("Item %s unexpected type: %s" % - (item, type(item))) - # If it's a module derive objects from it if we can. - if module is not None: - for (_name, obj) in _get_members(module, exclude_hidden): - if is_subclass(obj, base_cls): - derived.add(obj) - return derived - - -def get_member_names(obj, exclude_hidden=True): - """Get all the member names for a object.""" - return [name for (name, _obj) in _get_members(obj, exclude_hidden)] - - -def get_class_name(obj, fully_qualified=True): - """Get class name for object. - - If object is a type, fully qualified name of the type is returned. - Else, fully qualified name of the type of the object is returned. - For builtin types, just name is returned. - """ - if not isinstance(obj, six.class_types): - obj = type(obj) - try: - built_in = obj.__module__ in _BUILTIN_MODULES - except AttributeError: - pass - else: - if built_in: - try: - return obj.__qualname__ - except AttributeError: - return obj.__name__ - pieces = [] - try: - pieces.append(obj.__qualname__) - except AttributeError: - pieces.append(obj.__name__) - if fully_qualified: - try: - pieces.insert(0, obj.__module__) - except AttributeError: - pass - return '.'.join(pieces) - - -def get_all_class_names(obj, up_to=object): - """Get class names of object parent classes. - - Iterate over all class names object is instance or subclass of, - in order of method resolution (mro). If up_to parameter is provided, - only name of classes that are sublcasses to that class are returned. - """ - if not isinstance(obj, six.class_types): - obj = type(obj) - for cls in obj.mro(): - if issubclass(cls, up_to): - yield get_class_name(cls) - - -def get_callable_name(function): - """Generate a name from callable. - - Tries to do the best to guess fully qualified callable name. - """ - method_self = get_method_self(function) - if method_self is not None: - # This is a bound method. - if isinstance(method_self, six.class_types): - # This is a bound class method. - im_class = method_self - else: - im_class = type(method_self) - try: - parts = (im_class.__module__, function.__qualname__) - except AttributeError: - parts = (im_class.__module__, im_class.__name__, function.__name__) - elif inspect.ismethod(function) or inspect.isfunction(function): - # This could be a function, a static method, a unbound method... - try: - parts = (function.__module__, function.__qualname__) - except AttributeError: - if hasattr(function, 'im_class'): - # This is a unbound method, which exists only in python 2.x - im_class = function.im_class - parts = (im_class.__module__, - im_class.__name__, function.__name__) - else: - parts = (function.__module__, function.__name__) - else: - im_class = type(function) - if im_class is _TYPE_TYPE: - im_class = function - try: - parts = (im_class.__module__, im_class.__qualname__) - except AttributeError: - parts = (im_class.__module__, im_class.__name__) - return '.'.join(parts) - - -def get_method_self(method): - if not inspect.ismethod(method): - return None - try: - return six.get_method_self(method) - except AttributeError: - return None - - -def is_same_callback(callback1, callback2, strict=True): - """Returns if the two callbacks are the same.""" - if callback1 is callback2: - # This happens when plain methods are given (or static/non-bound - # methods). - return True - if callback1 == callback2: - if not strict: - return True - # Two bound methods are equal if functions themselves are equal and - # objects they are applied to are equal. This means that a bound - # method could be the same bound method on another object if the - # objects have __eq__ methods that return true (when in fact it is a - # different bound method). Python u so crazy! - try: - self1 = six.get_method_self(callback1) - self2 = six.get_method_self(callback2) - return self1 is self2 - except AttributeError: - pass - return False - - -def is_bound_method(method): - """Returns if the given method is bound to an object.""" - return bool(get_method_self(method)) - - -def is_subclass(obj, cls): - """Returns if the object is class and it is subclass of a given class.""" - return inspect.isclass(obj) and issubclass(obj, cls) - - -def _get_arg_spec(function): - if isinstance(function, type): - bound = True - function = function.__init__ - elif isinstance(function, (types.FunctionType, types.MethodType)): - bound = is_bound_method(function) - function = getattr(function, '__wrapped__', function) - else: - function = function.__call__ - bound = is_bound_method(function) - return inspect.getargspec(function), bound - - -def get_callable_args(function, required_only=False): - """Get names of callable arguments. - - Special arguments (like ``*args`` and ``**kwargs``) are not included into - output. - - If required_only is True, optional arguments (with default values) - are not included into output. - """ - argspec, bound = _get_arg_spec(function) - f_args = argspec.args - if required_only and argspec.defaults: - f_args = f_args[:-len(argspec.defaults)] - if bound: - f_args = f_args[1:] - return f_args - - -def accepts_kwargs(function): - """Returns True if function accepts kwargs.""" - argspec, _bound = _get_arg_spec(function) - return bool(argspec.keywords) From 2b959daf9e608b176f3af5faa185b7b5888ac269 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 15 Dec 2014 16:16:42 -0800 Subject: [PATCH 164/240] Add an example which shows how to send events out from tasks Tasks support a notification like channel that they can use to emit information that has occurred internal to then be received by any attached listeners. This kind of notification even works when ran remotely; so this example shows how to use that system to do something useful. Part of blueprint more-examples Change-Id: I104fa55e6b511df77464e3b89ee2bad6438482dd --- doc/source/examples.rst | 12 +++ taskflow/examples/wbe_event_sender.py | 148 ++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 taskflow/examples/wbe_event_sender.py diff --git a/doc/source/examples.rst b/doc/source/examples.rst index 24a43dd0..2df2dc34 100644 --- a/doc/source/examples.rst +++ b/doc/source/examples.rst @@ -188,6 +188,18 @@ Distributed execution (simple) :linenos: :lines: 16- +Distributed notification (simple) +================================= + +.. note:: + + Full source located at :example:`wbe_event_sender` + +.. literalinclude:: ../../taskflow/examples/wbe_event_sender.py + :language: python + :linenos: + :lines: 16- + Distributed mandelbrot (complex) ================================ diff --git a/taskflow/examples/wbe_event_sender.py b/taskflow/examples/wbe_event_sender.py new file mode 100644 index 00000000..38b6bfd9 --- /dev/null +++ b/taskflow/examples/wbe_event_sender.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import string +import sys +import time + +top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, + os.pardir)) +sys.path.insert(0, top_dir) + +from six.moves import range as compat_range + +from taskflow import engines +from taskflow.engines.worker_based import worker +from taskflow.patterns import linear_flow as lf +from taskflow import task +from taskflow.types import notifier +from taskflow.utils import threading_utils + +# INTRO: This examples shows how to use a remote workers event notification +# attribute to proxy back task event notifications to the controlling process. +# +# In this case a simple set of events are triggered by a worker running a +# task (simulated to be remote by using a kombu memory transport and threads). +# Those events that the 'remote worker' produces will then be proxied back to +# the task that the engine is running 'remotely', and then they will be emitted +# back to the original callbacks that exist in the originating engine +# process/thread. This creates a one-way *notification* channel that can +# transparently be used in-process, outside-of-process using remote workers and +# so-on that allows tasks to signal to its controlling process some sort of +# action that has occurred that the task may need to tell others about (for +# example to trigger some type of response when the task reaches 50% done...). + + +def event_receiver(event_type, details): + """This is the callback that (in this example) doesn't do much...""" + print("Recieved event '%s'" % event_type) + print("Details = %s" % details) + + +class EventReporter(task.Task): + """This is the task that will be running 'remotely' (not really remote).""" + + EVENTS = tuple(string.ascii_uppercase) + EVENT_DELAY = 0.1 + + def execute(self): + for i, e in enumerate(self.EVENTS): + details = { + 'leftover': self.EVENTS[i:], + } + self.notifier.notify(e, details) + time.sleep(self.EVENT_DELAY) + + +BASE_SHARED_CONF = { + 'exchange': 'taskflow', + 'transport': 'memory', + 'transport_options': { + 'polling_interval': 0.1, + }, +} + +# Until https://github.com/celery/kombu/issues/398 is resolved it is not +# recommended to run many worker threads in this example due to the types +# of errors mentioned in that issue. +MEMORY_WORKERS = 1 +WORKER_CONF = { + 'tasks': [ + # Used to locate which tasks we can run (we don't want to allow + # arbitrary code/tasks to be ran by any worker since that would + # open up a variety of vulnerabilities). + '%s:EventReporter' % (__name__), + ], +} + + +def run(engine_options): + reporter = EventReporter() + reporter.notifier.register(notifier.Notifier.ANY, event_receiver) + flow = lf.Flow('event-reporter').add(reporter) + eng = engines.load(flow, engine='worker-based', **engine_options) + eng.run() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.ERROR) + + # Setup our transport configuration and merge it into the worker and + # engine configuration so that both of those objects use it correctly. + worker_conf = dict(WORKER_CONF) + worker_conf.update(BASE_SHARED_CONF) + engine_options = dict(BASE_SHARED_CONF) + workers = [] + + # These topics will be used to request worker information on; those + # workers will respond with there capabilities which the executing engine + # will use to match pending tasks to a matched worker, this will cause + # the task to be sent for execution, and the engine will wait until it + # is finished (a response is recieved) and then the engine will either + # continue with other tasks, do some retry/failure resolution logic or + # stop (and potentially re-raise the remote workers failure)... + worker_topics = [] + + try: + # Create a set of worker threads to simulate actual remote workers... + print('Running %s workers.' % (MEMORY_WORKERS)) + for i in compat_range(0, MEMORY_WORKERS): + # Give each one its own unique topic name so that they can + # correctly communicate with the engine (they will all share the + # same exchange). + worker_conf['topic'] = 'worker-%s' % (i + 1) + worker_topics.append(worker_conf['topic']) + w = worker.Worker(**worker_conf) + runner = threading_utils.daemon_thread(w.run) + runner.start() + w.wait() + workers.append((runner, w.stop)) + + # Now use those workers to do something. + print('Executing some work.') + engine_options['topics'] = worker_topics + result = run(engine_options) + print('Execution finished.') + finally: + # And cleanup. + print('Stopping workers.') + while workers: + r, stopper = workers.pop() + stopper() + r.join() From dbc890f928ecfea3ee306b5bd0c7053eeb5070c9 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 18 Dec 2014 22:22:21 -0800 Subject: [PATCH 165/240] Correctly trigger 'on_exit' of starting/initial state Instead of not calling the 'on_exit' of the initialized and/or starting state we should make an attempt to call it if a function exists/was provided. This function will be called on the first event to be processed (which will cause the state machine to transition out of the starting state to a new stable state). Fixes bug 1404124 Change-Id: I037439313f9071af23c0859a62832d735f9abcd8 --- taskflow/tests/unit/test_types.py | 4 +++- taskflow/types/fsm.py | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/taskflow/tests/unit/test_types.py b/taskflow/tests/unit/test_types.py index 395d7d9b..28b57251 100644 --- a/taskflow/tests/unit/test_types.py +++ b/taskflow/tests/unit/test_types.py @@ -304,7 +304,9 @@ class FSMTest(test.TestCase): m.process_event('fall') self.assertEqual([('down', 'beat'), ('up', 'jump'), ('down', 'fall')], enter_transitions) - self.assertEqual([('down', 'jump'), ('up', 'fall')], exit_transitions) + self.assertEqual( + [('start', 'beat'), ('down', 'jump'), ('up', 'fall')], + exit_transitions) def test_run_iter(self): up_downs = [] diff --git a/taskflow/types/fsm.py b/taskflow/types/fsm.py index 9cf94d7b..2519e840 100644 --- a/taskflow/types/fsm.py +++ b/taskflow/types/fsm.py @@ -196,7 +196,12 @@ class FSM(object): if self._states[self._start_state]['terminal']: raise excp.InvalidState("Can not start from a terminal" " state '%s'" % (self._start_state)) - self._current = _Jump(self._start_state, None, None) + # No on enter will be called, since we are priming the state machine + # and have not really transitioned from anything to get here, we will + # though allow 'on_exit' to be called on the event that causes this + # to be moved from... + self._current = _Jump(self._start_state, None, + self._states[self._start_state]['on_exit']) def run(self, event, initialize=True): """Runs the state machine, using reactions only.""" From d0edb62b9a7da8c64737e95296983978fcb39cf1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 19 Dec 2014 16:25:30 -0800 Subject: [PATCH 166/240] Move the persistence base to the parent directory In order to match the directory/module layout of the other pluggable backends better move the persistence base module that defines the base abstract classes up into the parent directory. This makes it easier to look at the taskflow code-base and understand the common layout. Change-Id: I7887cb0241b8fe65cbdfee32c101c3df5f05d27c --- doc/source/persistence.rst | 8 ++++---- taskflow/persistence/backends/impl_dir.py | 2 +- taskflow/persistence/backends/impl_memory.py | 2 +- taskflow/persistence/backends/impl_sqlalchemy.py | 2 +- taskflow/persistence/backends/impl_zookeeper.py | 2 +- taskflow/persistence/{backends => }/base.py | 0 6 files changed, 8 insertions(+), 8 deletions(-) rename taskflow/persistence/{backends => }/base.py (100%) diff --git a/doc/source/persistence.rst b/doc/source/persistence.rst index 0da68de1..ce35d357 100644 --- a/doc/source/persistence.rst +++ b/doc/source/persistence.rst @@ -38,7 +38,7 @@ How it is used On :doc:`engine ` construction typically a backend (it can be optional) will be provided which satisfies the -:py:class:`~taskflow.persistence.backends.base.Backend` abstraction. Along with +:py:class:`~taskflow.persistence.base.Backend` abstraction. Along with providing a backend object a :py:class:`~taskflow.persistence.logbook.FlowDetail` object will also be created and provided (this object will contain the details about the flow to be @@ -55,7 +55,7 @@ interface to the underlying backend storage objects (it provides helper functions that are commonly used by the engine, avoiding repeating code when interacting with the provided :py:class:`~taskflow.persistence.logbook.FlowDetail` and -:py:class:`~taskflow.persistence.backends.base.Backend` objects). As an engine +:py:class:`~taskflow.persistence.base.Backend` objects). As an engine initializes it will extract (or create) :py:class:`~taskflow.persistence.logbook.AtomDetail` objects for each atom in the workflow the engine will be executing. @@ -72,7 +72,7 @@ predecessor :py:class:`~taskflow.persistence.logbook.AtomDetail` outputs and states (which may have been persisted in a past run). This will result in either using there previous information or by running those predecessors and saving their output to the :py:class:`~taskflow.persistence.logbook.FlowDetail` -and :py:class:`~taskflow.persistence.backends.base.Backend` objects. This +and :py:class:`~taskflow.persistence.base.Backend` objects. This execution, analysis and interaction with the storage objects continues (what is described here is a simplification of what really happens; which is quite a bit more complex) until the engine has finished running (at which point the engine @@ -248,7 +248,7 @@ Interfaces ========== .. automodule:: taskflow.persistence.backends -.. automodule:: taskflow.persistence.backends.base +.. automodule:: taskflow.persistence.base .. automodule:: taskflow.persistence.logbook Hierarchy diff --git a/taskflow/persistence/backends/impl_dir.py b/taskflow/persistence/backends/impl_dir.py index 0a687473..42bd5442 100644 --- a/taskflow/persistence/backends/impl_dir.py +++ b/taskflow/persistence/backends/impl_dir.py @@ -24,7 +24,7 @@ import six from taskflow import exceptions as exc from taskflow import logging -from taskflow.persistence.backends import base +from taskflow.persistence import base from taskflow.persistence import logbook from taskflow.utils import lock_utils from taskflow.utils import misc diff --git a/taskflow/persistence/backends/impl_memory.py b/taskflow/persistence/backends/impl_memory.py index 6c58718a..79024e87 100644 --- a/taskflow/persistence/backends/impl_memory.py +++ b/taskflow/persistence/backends/impl_memory.py @@ -19,7 +19,7 @@ import six from taskflow import exceptions as exc from taskflow import logging -from taskflow.persistence.backends import base +from taskflow.persistence import base from taskflow.persistence import logbook LOG = logging.getLogger(__name__) diff --git a/taskflow/persistence/backends/impl_sqlalchemy.py b/taskflow/persistence/backends/impl_sqlalchemy.py index c65c9134..72930e22 100644 --- a/taskflow/persistence/backends/impl_sqlalchemy.py +++ b/taskflow/persistence/backends/impl_sqlalchemy.py @@ -33,9 +33,9 @@ from sqlalchemy import pool as sa_pool from taskflow import exceptions as exc from taskflow import logging -from taskflow.persistence.backends import base from taskflow.persistence.backends.sqlalchemy import migration from taskflow.persistence.backends.sqlalchemy import models +from taskflow.persistence import base from taskflow.persistence import logbook from taskflow.types import failure from taskflow.utils import async_utils diff --git a/taskflow/persistence/backends/impl_zookeeper.py b/taskflow/persistence/backends/impl_zookeeper.py index ca801b43..0466d77f 100644 --- a/taskflow/persistence/backends/impl_zookeeper.py +++ b/taskflow/persistence/backends/impl_zookeeper.py @@ -22,7 +22,7 @@ from oslo.serialization import jsonutils from taskflow import exceptions as exc from taskflow import logging -from taskflow.persistence.backends import base +from taskflow.persistence import base from taskflow.persistence import logbook from taskflow.utils import kazoo_utils as k_utils from taskflow.utils import misc diff --git a/taskflow/persistence/backends/base.py b/taskflow/persistence/base.py similarity index 100% rename from taskflow/persistence/backends/base.py rename to taskflow/persistence/base.py From 2a8fde1798dbcaab53d0a1aba9740f729ba64f83 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 12 Sep 2014 18:49:29 -0700 Subject: [PATCH 167/240] Get the basics of a process executor working Since we support various executors (threaded and distributed) the next best executor when a threaded executor will not perform and a distributed one requires to much setup is a local process based one so it would be great to support this where we can. Things that are currently (likely never) not going to work: * Non-pickleable/non-copyable tasks * Tasks that return non-pickleable/non-copyable results * Tasks that use non-pickleable/non-copyable args/kwargs Part of blueprint process-executor Change-Id: I966ae01d390c7217b858db3feb2db949ce5c08d1 --- doc/source/conf.py | 1 + doc/source/engines.rst | 22 ++++++-- taskflow/engines/action_engine/engine.py | 28 +++++++-- taskflow/engines/action_engine/executor.py | 66 +++++++++++++++++----- taskflow/engines/worker_based/engine.py | 4 +- taskflow/engines/worker_based/executor.py | 18 ++---- taskflow/tests/unit/test_engines.py | 40 +++++++++++++ taskflow/utils/threading_utils.py | 7 +++ 8 files changed, 147 insertions(+), 39 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 2fb1e7ec..9dec3b69 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -56,6 +56,7 @@ modindex_common_prefix = ['taskflow.'] # Shortened external links. extlinks = { 'example': (source_tree + '/taskflow/examples/%s.py', ''), + 'pybug': ('http://bugs.python.org/issue%s', ''), } # -- Options for HTML output -------------------------------------------------- diff --git a/doc/source/engines.rst b/doc/source/engines.rst index 0c4b822f..474fa666 100644 --- a/doc/source/engines.rst +++ b/doc/source/engines.rst @@ -160,7 +160,8 @@ Parallel **Engine type**: ``'parallel'`` -Parallel engine schedules tasks onto different threads to run them in parallel. +A parallel engine schedules tasks onto different threads/processes to allow for +running non-dependent tasks simultaneously. Additional supported keyword arguments: @@ -168,17 +169,24 @@ Additional supported keyword arguments: interface; it will be used for scheduling tasks. You can use instances of a `thread pool executor`_ or a :py:class:`green executor ` (which internally uses - `eventlet `_ and greenthread pools). + `eventlet `_ and greenthread pools) or a + `process pool executor`_. .. tip:: - Sharing executor between engine instances provides better - scalability by reducing thread creation and teardown as well as by reusing - existing pools (which is a good practice in general). + Sharing an executor between engine instances provides better + scalability by reducing thread/process creation and teardown as well as by + reusing existing pools (which is a good practice in general). .. note:: - Running tasks with a `process pool executor`_ is not currently supported. + Running tasks with a `process pool executor`_ is **experimentally** + supported. This is mainly due to the `futures backport`_ and + the `multiprocessing`_ module that exist in older versions of python not + being as up to date (with important fixes such as :pybug:`4892`, + :pybug:`6721`, :pybug:`9205`, :pybug:`11635`, :pybug:`16284`, + :pybug:`22393` and others...) as the most recent python version (which + themselves have a variety of ongoing/recent bugs). Worker-based ------------ @@ -347,8 +355,10 @@ Hierarchy taskflow.engines.worker_based.engine.WorkerBasedActionEngine :parts: 1 +.. _multiprocessing: https://docs.python.org/2/library/multiprocessing.html .. _future: https://docs.python.org/dev/library/concurrent.futures.html#future-objects .. _executor: https://docs.python.org/dev/library/concurrent.futures.html#concurrent.futures.Executor .. _networkx: https://networkx.github.io/ +.. _futures backport: https://pypi.python.org/pypi/futures .. _thread pool executor: https://docs.python.org/dev/library/concurrent.futures.html#threadpoolexecutor .. _process pool executor: https://docs.python.org/dev/library/concurrent.futures.html#processpoolexecutor diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index ffc3a80a..118d9dd3 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -14,9 +14,11 @@ # License for the specific language governing permissions and limitations # under the License. +import abc import contextlib import threading +from concurrent import futures from oslo.utils import excutils from taskflow.engines.action_engine import compiler @@ -58,7 +60,6 @@ class ActionEngine(base.Engine): the tasks and flow being ran can go through. """ _compiler_factory = compiler.PatternCompiler - _task_executor_factory = executor.SerialTaskExecutor def __init__(self, flow, flow_detail, backend, options): super(ActionEngine, self).__init__(flow, flow_detail, backend, options) @@ -202,9 +203,10 @@ class ActionEngine(base.Engine): self._runtime.reset_all() self._change_state(states.PENDING) - @misc.cachedproperty + @abc.abstractproperty def _task_executor(self): return self._task_executor_factory() + pass @misc.cachedproperty def _compiler(self): @@ -226,12 +228,26 @@ class SerialActionEngine(ActionEngine): """Engine that runs tasks in serial manner.""" _storage_factory = atom_storage.SingleThreadedStorage + @misc.cachedproperty + def _task_executor(self): + return executor.SerialTaskExecutor() + class ParallelActionEngine(ActionEngine): """Engine that runs tasks in parallel manner.""" _storage_factory = atom_storage.MultiThreadedStorage - def _task_executor_factory(self): - return executor.ParallelTaskExecutor( - executor=self._options.get('executor'), - max_workers=self._options.get('max_workers')) + @misc.cachedproperty + def _task_executor(self): + kwargs = { + 'executor': self._options.get('executor'), + 'max_workers': self._options.get('max_workers'), + } + # The reason we use the library/built-in futures is to allow for + # instances of that to be detected and handled correctly, instead of + # forcing everyone to use our derivatives... + if isinstance(kwargs['executor'], futures.ProcessPoolExecutor): + executor_cls = executor.ParallelProcessTaskExecutor + else: + executor_cls = executor.ParallelThreadTaskExecutor + return executor_cls(**kwargs) diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index 002068b3..97756f54 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -75,7 +75,8 @@ class TaskExecutor(object): """ @abc.abstractmethod - def execute_task(self, task, task_uuid, arguments, progress_callback=None): + def execute_task(self, task, task_uuid, arguments, + progress_callback=None): """Schedules task execution.""" @abc.abstractmethod @@ -128,32 +129,69 @@ class ParallelTaskExecutor(TaskExecutor): def __init__(self, executor=None, max_workers=None): self._executor = executor self._max_workers = max_workers - self._create_executor = executor is None + self._own_executor = executor is None - def execute_task(self, task, task_uuid, arguments, progress_callback=None): - fut = self._executor.submit(_execute_task, - task, arguments, - progress_callback=progress_callback) + @abc.abstractmethod + def _create_executor(self, max_workers=None): + """Called when an executor has not been provided to make one.""" + + def _submit_task(self, func, task, *args, **kwargs): + fut = self._executor.submit(func, task, *args, **kwargs) fut.atom = task return fut + def execute_task(self, task, task_uuid, arguments, progress_callback=None): + return self._submit_task(_execute_task, task, arguments, + progress_callback=progress_callback) + def revert_task(self, task, task_uuid, arguments, result, failures, progress_callback=None): - fut = self._executor.submit(_revert_task, - task, arguments, result, failures, - progress_callback=progress_callback) - fut.atom = task - return fut + return self._submit_task(_revert_task, task, arguments, result, + failures, progress_callback=progress_callback) def start(self): - if self._create_executor: + if self._own_executor: if self._max_workers is not None: max_workers = self._max_workers else: max_workers = threading_utils.get_optimal_thread_count() - self._executor = futures.ThreadPoolExecutor(max_workers) + self._executor = self._create_executor(max_workers=max_workers) def stop(self): - if self._create_executor: + if self._own_executor: self._executor.shutdown(wait=True) self._executor = None + + +class ParallelThreadTaskExecutor(ParallelTaskExecutor): + """Executes tasks in parallel using a thread pool executor.""" + + def _create_executor(self, max_workers=None): + return futures.ThreadPoolExecutor(max_workers=max_workers) + + +class ParallelProcessTaskExecutor(ParallelTaskExecutor): + """Executes tasks in parallel using a process pool executor. + + NOTE(harlowja): this executor executes tasks in external processes, so that + implies that tasks that are sent to that external process are pickleable + since this is how the multiprocessing works (sending pickled objects back + and forth). + """ + + def _create_executor(self, max_workers=None): + return futures.ProcessPoolExecutor(max_workers=max_workers) + + def _submit_task(self, func, task, *args, **kwargs): + """Submit a function to run the given task (with given args/kwargs). + + NOTE(harlowja): task callbacks/notifications will not currently + work (they will be removed before being sent to the target process + for execution). + """ + kwargs.pop('progress_callback', None) + clone = task.copy(retain_listeners=False) + fut = super(ParallelProcessTaskExecutor, self)._submit_task( + func, clone, *args, **kwargs) + fut.atom = task + return fut diff --git a/taskflow/engines/worker_based/engine.py b/taskflow/engines/worker_based/engine.py index aefce23f..df915fc9 100644 --- a/taskflow/engines/worker_based/engine.py +++ b/taskflow/engines/worker_based/engine.py @@ -18,6 +18,7 @@ from taskflow.engines.action_engine import engine from taskflow.engines.worker_based import executor from taskflow.engines.worker_based import protocol as pr from taskflow import storage as t_storage +from taskflow.utils import misc class WorkerBasedActionEngine(engine.ActionEngine): @@ -44,7 +45,8 @@ class WorkerBasedActionEngine(engine.ActionEngine): _storage_factory = t_storage.SingleThreadedStorage - def _task_executor_factory(self): + @misc.cachedproperty + def _task_executor(self): try: return self._options['executor'] except KeyError: diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index bdef7bff..2827fb5f 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -34,12 +34,6 @@ from taskflow.utils import threading_utils as tu LOG = logging.getLogger(__name__) -def _is_alive(thread): - if not thread: - return False - return thread.is_alive() - - class PeriodicWorker(object): """Calls a set of functions when activated periodically. @@ -181,7 +175,7 @@ class WorkerTaskExecutor(executor.TaskExecutor): self._requests_cache.cleanup(self._handle_expired_request) def _submit_task(self, task, task_uuid, action, arguments, - progress_callback, **kwargs): + progress_callback=None, **kwargs): """Submit task request to a worker.""" request = pr.Request(task, task_uuid, action, arguments, self._transition_timeout, **kwargs) @@ -239,13 +233,13 @@ class WorkerTaskExecutor(executor.TaskExecutor): def execute_task(self, task, task_uuid, arguments, progress_callback=None): return self._submit_task(task, task_uuid, pr.EXECUTE, arguments, - progress_callback) + progress_callback=progress_callback) def revert_task(self, task, task_uuid, arguments, result, failures, progress_callback=None): return self._submit_task(task, task_uuid, pr.REVERT, arguments, - progress_callback, result=result, - failures=failures) + progress_callback=progress_callback, + result=result, failures=failures) def wait_for_workers(self, workers=1, timeout=None): """Waits for geq workers to notify they are ready to do work. @@ -273,11 +267,11 @@ class WorkerTaskExecutor(executor.TaskExecutor): def start(self): """Starts proxy thread and associated topic notification thread.""" - if not _is_alive(self._proxy_thread): + if not tu.is_alive(self._proxy_thread): self._proxy_thread = tu.daemon_thread(self._proxy.start) self._proxy_thread.start() self._proxy.wait() - if not _is_alive(self._periodic_thread): + if not tu.is_alive(self._periodic_thread): self._periodic.reset() self._periodic_thread = tu.daemon_thread(self._periodic.start) self._periodic_thread.start() diff --git a/taskflow/tests/unit/test_engines.py b/taskflow/tests/unit/test_engines.py index 1ff81353..baa5e81f 100644 --- a/taskflow/tests/unit/test_engines.py +++ b/taskflow/tests/unit/test_engines.py @@ -17,6 +17,7 @@ import contextlib import testtools +from testtools import testcase import taskflow.engines from taskflow.engines.action_engine import engine as eng @@ -602,11 +603,50 @@ class ParallelEngineWithEventletTest(EngineTaskTest, def _make_engine(self, flow, flow_detail=None, executor=None): if executor is None: executor = futures.GreenThreadPoolExecutor() + self.addCleanup(executor.shutdown) return taskflow.engines.load(flow, flow_detail=flow_detail, backend=self.backend, engine='parallel', executor=executor) +class ParallelEngineWithProcessTest(EngineTaskTest, + EngineLinearFlowTest, + EngineParallelFlowTest, + EngineLinearAndUnorderedExceptionsTest, + EngineGraphFlowTest, + EngineCheckingTaskTest, + test.TestCase): + _SKIP_TYPES = (utils.SaveOrderTask,) + + def test_correct_load(self): + engine = self._make_engine(utils.TaskNoRequiresNoReturns) + self.assertIsInstance(engine, eng.ParallelActionEngine) + + def _make_engine(self, flow, flow_detail=None, executor=None): + if executor is None: + executor = futures.ProcessPoolExecutor(1) + self.addCleanup(executor.shutdown) + e = taskflow.engines.load(flow, flow_detail=flow_detail, + backend=self.backend, engine='parallel', + executor=executor) + # FIXME(harlowja): fix this so that we can actually tests these + # testcases, without having task/global test state that is retained + # and inspected; this doesn't work in a multi-process situation since + # the tasks execute in another process with its own memory/heap + # which this process later can't view/introspect... + try: + e.compile() + for a in e.compilation.execution_graph: + if isinstance(a, self._SKIP_TYPES): + baddies = [a.__name__ for a in self._SKIP_TYPES] + raise testcase.TestSkipped("Process engines can not" + " run flows that contain" + " %s tasks" % baddies) + except (TypeError, exc.TaskFlowException): + pass + return e + + class WorkerBasedEngineTest(EngineTaskTest, EngineLinearFlowTest, EngineParallelFlowTest, diff --git a/taskflow/utils/threading_utils.py b/taskflow/utils/threading_utils.py index b3749bca..5048401c 100644 --- a/taskflow/utils/threading_utils.py +++ b/taskflow/utils/threading_utils.py @@ -40,6 +40,13 @@ else: Event = threading.Event +def is_alive(thread): + """Helper to determine if a thread is alive (handles none safely).""" + if not thread: + return False + return thread.is_alive() + + def get_ident(): """Return the 'thread identifier' of the current thread.""" return _thread.get_ident() From a170f4bb9efdf6c40989b37ac60afbbaccd4c0df Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 20 Dec 2014 16:28:14 -0800 Subject: [PATCH 168/240] Move the engine scoping test to its engines test folder Since the scoping is an action_engine implementation detail its test should belong in the action_engine test cases folder instead of being at the top level unit test folder. Change-Id: Ic436c534103d1d9fafad95299bd2632cc7ee5634 --- .../test_scoping.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename taskflow/tests/unit/{test_action_engine_scoping.py => action_engine/test_scoping.py} (100%) diff --git a/taskflow/tests/unit/test_action_engine_scoping.py b/taskflow/tests/unit/action_engine/test_scoping.py similarity index 100% rename from taskflow/tests/unit/test_action_engine_scoping.py rename to taskflow/tests/unit/action_engine/test_scoping.py From e841b5a6c23588fa7058a4b6aa58370d57590200 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 14 Nov 2014 18:39:32 -0800 Subject: [PATCH 169/240] Get event/notification sending working correctly In order to support tasks notifications and progress updates we need to establish a channel & proxy by which those events can be sent from the process executing and producing those events and the originating process that requested that task to be executed. This review adds on such a proxy and adjusts a cloned tasks notification callbacks to place messages on a queue that will be picked up by a thread in the originating process for dispatch to the original callbacks that were registered with the non-cloned task (therefore making the original callbacks appear to be called as they are supposed to be). Part of blueprint process-executor Change-Id: I01c83f13186e4be9fa28c32e34e907bb133e8fb3 --- doc/source/examples.rst | 12 + taskflow/engines/action_engine/engine.py | 2 + taskflow/engines/action_engine/executor.py | 330 ++++++++++++++++++++- taskflow/examples/alphabet_soup.py | 101 +++++++ taskflow/utils/lock_utils.py | 9 +- 5 files changed, 443 insertions(+), 11 deletions(-) create mode 100644 taskflow/examples/alphabet_soup.py diff --git a/doc/source/examples.rst b/doc/source/examples.rst index 5043626d..338c3921 100644 --- a/doc/source/examples.rst +++ b/doc/source/examples.rst @@ -46,6 +46,18 @@ Building a car :linenos: :lines: 16- +Iterating over the alphabet (using processes) +============================================= + +.. note:: + + Full source located at :example:`alphabet_soup`. + +.. literalinclude:: ../../taskflow/examples/alphabet_soup.py + :language: python + :linenos: + :lines: 16- + Watching execution timing ========================= diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index 118d9dd3..3f39774b 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -248,6 +248,8 @@ class ParallelActionEngine(ActionEngine): # forcing everyone to use our derivatives... if isinstance(kwargs['executor'], futures.ProcessPoolExecutor): executor_cls = executor.ParallelProcessTaskExecutor + kwargs['dispatch_periodicity'] = self._options.get( + 'dispatch_periodicity') else: executor_cls = executor.ParallelThreadTaskExecutor return executor_cls(**kwargs) diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index 97756f54..5d063c5b 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -15,13 +15,24 @@ # under the License. import abc +import functools +import multiprocessing +import os +import pickle +import threading +from oslo.utils import excutils +from oslo.utils import timeutils import six +from six.moves import queue as compat_queue +from six.moves import range as compat_range +from taskflow import logging from taskflow import task as task_atom from taskflow.types import failure from taskflow.types import futures from taskflow.types import notifier +from taskflow.types import timing from taskflow.utils import async_utils from taskflow.utils import threading_utils @@ -29,10 +40,32 @@ from taskflow.utils import threading_utils EXECUTED = 'executed' REVERTED = 'reverted' +# See http://bugs.python.org/issue1457119 for why this is so complex... +_PICKLE_ERRORS = [pickle.PickleError, TypeError] +try: + import cPickle as _cPickle + _PICKLE_ERRORS.append(_cPickle.PickleError) +except ImportError: + pass +_PICKLE_ERRORS = tuple(_PICKLE_ERRORS) +_SEND_ERRORS = (IOError, EOFError) +_UPDATE_PROGRESS = task_atom.EVENT_UPDATE_PROGRESS + +LOG = logging.getLogger(__name__) + + +def _maybe_forever(limit=None): + if limit is None: + while True: + yield + else: + for i in compat_range(0, limit): + yield + def _execute_task(task, arguments, progress_callback=None): with notifier.register_deregister(task.notifier, - task_atom.EVENT_UPDATE_PROGRESS, + _UPDATE_PROGRESS, callback=progress_callback): try: task.pre_execute() @@ -51,7 +84,7 @@ def _revert_task(task, arguments, result, failures, progress_callback=None): arguments[task_atom.REVERT_RESULT] = result arguments[task_atom.REVERT_FLOW_FAILURES] = failures with notifier.register_deregister(task.notifier, - task_atom.EVENT_UPDATE_PROGRESS, + _UPDATE_PROGRESS, callback=progress_callback): try: task.pre_revert() @@ -65,6 +98,182 @@ def _revert_task(task, arguments, result, failures, progress_callback=None): return (REVERTED, result) +class _JoinedWorkItem(object): + """The piece of work that will executed by a process executor. + + This will call the target function, then wait until the queues items + have been completed (via calls to task_done) before offically being + finished. + + NOTE(harlowja): this is done so that the task function will *not* return + until all of its notifications have been proxied back to its originating + task. If we didn't do this then the executor would see this task as done + and then potentially start tasks that are successors of the task that just + finished even though notifications are still left to be sent from the + previously finished task... + """ + + def __init__(self, queue, func, task, *args, **kwargs): + self._queue = queue + self._func = func + self._task = task + self._args = args + self._kwargs = kwargs + + def __call__(self): + args = self._args + kwargs = self._kwargs + try: + return self._func(self._task, *args, **kwargs) + finally: + w = timing.StopWatch().start() + self._queue.join() + LOG.blather("Waited %0.2f seconds until task '%s' emitted" + " notifications were depleted", w.elapsed(), + self._task) + + +class _EventSender(object): + """Sends event information from a child worker process to its creator.""" + + def __init__(self, queue): + self._queue = queue + self._pid = None + + def __call__(self, event_type, details): + # NOTE(harlowja): this is done in late in execution to ensure that this + # happens in the child process and not the parent process (where the + # constructor is called). + if self._pid is None: + self._pid = os.getpid() + message = { + 'created_on': timeutils.utcnow(), + 'sender': { + 'pid': self._pid, + }, + 'body': { + 'event_type': event_type, + 'details': details, + }, + } + try: + self._queue.put(message) + except _PICKLE_ERRORS: + LOG.warn("Failed serializing message %s", message, exc_info=True) + except _SEND_ERRORS: + LOG.warn("Failed sending message %s", message, exc_info=True) + + +class _EventTarget(object): + """An immutable helper object that represents a target of an event.""" + + def __init__(self, future, task, queue): + self.future = future + self.task = task + self.queue = queue + + +class _EventDispatcher(object): + """Dispatches event information received from child worker processes.""" + + # When the run() method is busy (typically in a thread) we want to set + # these so that the thread can know how long to sleep when there is no + # active work to dispatch (when there is active targets, there queues + # will have amount/count of items removed before returning to work on + # the next target...) + _SPIN_PERIODICITY = 0.01 + _SPIN_DISPATCH_AMOUNT = 1 + + # TODO(harlowja): look again at using a non-polling mechanism that uses + # select instead of queues to achieve better ability to detect when + # messages are ready/available... + + def __init__(self, dispatch_periodicity=None): + if dispatch_periodicity is None: + dispatch_periodicity = self._SPIN_PERIODICITY + if dispatch_periodicity <= 0: + raise ValueError("Provided dispatch periodicity must be greater" + " than zero and not '%s'" % dispatch_periodicity) + self._targets = set() + self._dead = threading_utils.Event() + self._lock = threading.Lock() + self._periodicity = dispatch_periodicity + self._stop_when_empty = False + + def register(self, target): + with self._lock: + self._targets.add(target) + + def _dispatch_until_empty(self, target, limit=None): + it = _maybe_forever(limit=limit) + while True: + try: + six.next(it) + except StopIteration: + break + else: + try: + message = target.queue.get_nowait() + except compat_queue.Empty: + break + else: + try: + self._dispatch(target.task, message) + finally: + target.queue.task_done() + + def deregister(self, target): + with self._lock: + try: + self._targets.remove(target) + except KeyError: + pass + + def reset(self): + self._stop_when_empty = False + self._dead.clear() + + def interrupt(self): + self._stop_when_empty = True + self._dead.set() + + def _dispatch(self, task, message): + LOG.blather("Dispatching message %s to task '%s'", message, task) + body = message['body'] + task.notifier.notify(body['event_type'], body['details']) + + def _dispatch_iter(self, targets): + # A generator that yields at certain points to allow the main run() + # method to use this to dispatch in iterations (and also allows it + # to check if it has been stopped between iterations). + for target in targets: + if target not in self._targets: + # Must of been removed... + continue + # NOTE(harlowja): Limits are used here to avoid one + # task unequally dispatching, this forces round-robin + # like behavior... + self._dispatch_until_empty(target, + limit=self._SPIN_DISPATCH_AMOUNT) + yield target + + def run(self): + w = timing.StopWatch(duration=self._periodicity) + while (not self._dead.is_set() or + (self._stop_when_empty and self._targets)): + w.restart() + with self._lock: + targets = self._targets.copy() + for _target in self._dispatch_iter(targets): + if self._stop_when_empty: + continue + if self._dead.is_set(): + break + leftover = w.leftover() + if leftover: + self._dead.wait(leftover) + + @six.add_metaclass(abc.ABCMeta) class TaskExecutor(object): """Executes and reverts tasks. @@ -176,22 +385,125 @@ class ParallelProcessTaskExecutor(ParallelTaskExecutor): NOTE(harlowja): this executor executes tasks in external processes, so that implies that tasks that are sent to that external process are pickleable since this is how the multiprocessing works (sending pickled objects back - and forth). + and forth) and that the bound handlers (for progress updating in + particular) are proxied correctly from that external process to the one + that is alive in the parent process to ensure that callbacks registered in + the parent are executed on events in the child. """ + def __init__(self, executor=None, max_workers=None, + dispatch_periodicity=None): + super(ParallelProcessTaskExecutor, self).__init__( + executor=executor, max_workers=max_workers) + self._manager = multiprocessing.Manager() + self._queue_factory = lambda: self._manager.JoinableQueue() + self._dispatcher = _EventDispatcher( + dispatch_periodicity=dispatch_periodicity) + self._worker = None + def _create_executor(self, max_workers=None): return futures.ProcessPoolExecutor(max_workers=max_workers) + def start(self): + super(ParallelProcessTaskExecutor, self).start() + if not threading_utils.is_alive(self._worker): + self._dispatcher.reset() + self._worker = threading_utils.daemon_thread(self._dispatcher.run) + self._worker.start() + + def stop(self): + self._dispatcher.interrupt() + super(ParallelProcessTaskExecutor, self).stop() + if threading_utils.is_alive(self._worker): + self._worker.join() + self._worker = None + self._dispatcher.reset() + + def _rebind_task(self, task, clone, queue, progress_callback=None): + # Creates and binds proxies for all events the task could receive + # so that when the clone runs in another process that this task + # can recieve the same notifications (thus making it look like the + # the notifications are transparently happening in this process). + needed = set() + for (event_type, listeners) in task.notifier.listeners_iter(): + if listeners: + needed.add(event_type) + # We don't register for the 'ANY' event; since that meta event type + # will be correctly proxied by the task notifier directly without + # needing clone replication. + needed.discard(task.notifier.ANY) + if progress_callback is not None: + needed.add(_UPDATE_PROGRESS) + for event_type in needed: + clone.notifier.register(event_type, _EventSender(queue)) + return needed + def _submit_task(self, func, task, *args, **kwargs): """Submit a function to run the given task (with given args/kwargs). - NOTE(harlowja): task callbacks/notifications will not currently - work (they will be removed before being sent to the target process - for execution). + NOTE(harlowja): Adjust all events to be proxies instead since we want + those callbacks to be activated in this process, not in the child, + also since typically callbacks are functors (or callables) we can + not pickle those in the first place... + + To make sure people understand how this works, the following is a + lengthy description of what is going on here, read at will: + + So to ensure that we are proxying task triggered events that occur + in the executed subprocess (which will be created and used by the + thing using the multiprocessing based executor) we need to establish + a link between that process and this process that ensures that when a + event is triggered in that task in that process that a corresponding + event is triggered on the original task that was requested to be ran + in this process. + + To accomplish this we have to create a copy of the task (without + any listeners) and then reattach a new set of listeners that will + now instead of calling the desired listeners just place messages + for this process (a dispatcher thread that is created in this class) + to dispatch to the original task (using a per task queue that is used + and associated to know which task to proxy back too, since it is + possible that there many be *many* subprocess running at the same + time, each running a different task). + + Once the subprocess task has finished execution, the executor will + then trigger a callback (``on_done`` in this case) that will remove + the task + queue from the dispatcher (which will stop any further + proxying back to the original task). """ - kwargs.pop('progress_callback', None) + progress_callback = kwargs.pop('progress_callback', None) clone = task.copy(retain_listeners=False) - fut = super(ParallelProcessTaskExecutor, self)._submit_task( - func, clone, *args, **kwargs) + queue = self._queue_factory() + bound = self._rebind_task(task, clone, queue, + progress_callback=progress_callback) + LOG.blather("Bound %s event types to clone of '%s'", bound, task) + if progress_callback is not None: + binder = functools.partial(task.notifier.register, + _UPDATE_PROGRESS, progress_callback) + unbinder = functools.partial(task.notifier.deregister, + _UPDATE_PROGRESS, progress_callback) + else: + binder = unbinder = lambda: None + + # Ensure the target task (not the clone) is ready and able to receive + # dispatched messages (and start the dispatching process by + # registering) with the dispatcher. + binder() + work = _JoinedWorkItem(queue, func, clone, *args, **kwargs) + try: + fut = self._executor.submit(work) + except RuntimeError: + with excutils.save_and_reraise_exception(): + unbinder() + + # This will trigger the proxying to begin... + target = _EventTarget(fut, task, queue) + self._dispatcher.register(target) + + def on_done(unbinder, target, fut): + self._dispatcher.deregister(target) + unbinder() + fut.atom = task + fut.add_done_callback(functools.partial(on_done, unbinder, target)) return fut diff --git a/taskflow/examples/alphabet_soup.py b/taskflow/examples/alphabet_soup.py new file mode 100644 index 00000000..4e6a0ff9 --- /dev/null +++ b/taskflow/examples/alphabet_soup.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import fractions +import functools +import logging +import os +import string +import sys +import time + +from concurrent import futures + +logging.basicConfig(level=logging.ERROR) + +self_dir = os.path.abspath(os.path.dirname(__file__)) +top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, + os.pardir)) +sys.path.insert(0, top_dir) +sys.path.insert(0, self_dir) + +from taskflow import engines +from taskflow import exceptions +from taskflow.patterns import linear_flow +from taskflow import task + + +# In this example we show how a simple linear set of tasks can be executed +# using local processes (and not threads or remote workers) with minimial (if +# any) modification to those tasks to make them safe to run in this mode. +# +# This is useful since it allows further scaling up your workflows when thread +# execution starts to become a bottleneck (which it can start to be due to the +# GIL in python). It also offers a intermediary scalable runner that can be +# used when the scale and or setup of remote workers is not desirable. + +# How many local processes to potentially use when executing... (one is fine +# for this example, but more can be used to show play around with what happens +# with many...) +WORKERS = 1 + + +def progress_printer(task, event_type, details): + # This callback, attached to each task will be called in the local + # process (not the child processes)... + progress = details.pop('progress') + progress = int(progress * 100.0) + print("Task '%s' reached %d%% completion" % (task.name, progress)) + + +class AlphabetTask(task.Task): + # Second delay between each progress part. + _DELAY = 0.1 + + # This task will run in X main stages (each with a different progress + # report that will be delivered back to the running process...). The + # initial 0% and 100% are triggered automatically by the engine when + # a task is started and finished (so that's why those are not emitted + # here). + _PROGRESS_PARTS = [fractions.Fraction("%s/5" % x) for x in range(1, 5)] + + def execute(self): + for p in self._PROGRESS_PARTS: + self.update_progress(p) + time.sleep(self._DELAY) + + +print("Constructing...") +soup = linear_flow.Flow("alphabet-soup") +for letter in string.ascii_lowercase: + abc = AlphabetTask(letter) + abc.notifier.register(task.EVENT_UPDATE_PROGRESS, + functools.partial(progress_printer, abc)) + soup.add(abc) +try: + with futures.ProcessPoolExecutor(WORKERS) as executor: + print("Loading...") + e = engines.load(soup, engine='parallel', executor=executor) + print("Compiling...") + e.compile() + print("Preparing...") + e.prepare() + print("Running...") + e.run() + print("Done...") +except exceptions.NotImplementedError as e: + print(e) diff --git a/taskflow/utils/lock_utils.py b/taskflow/utils/lock_utils.py index b61668b0..b74931e9 100644 --- a/taskflow/utils/lock_utils.py +++ b/taskflow/utils/lock_utils.py @@ -37,8 +37,13 @@ LOG = logging.getLogger(__name__) @contextlib.contextmanager def try_lock(lock): - """Attempts to acquire a lock, and autoreleases if acquisition occurred.""" - was_locked = lock.acquire(blocking=False) + """Attempts to acquire a lock, and auto releases if acquired (on exit).""" + # NOTE(harlowja): the keyword argument for 'blocking' does not work + # in py2.x and only is fixed in py3.x (this adjustment is documented + # and/or debated in http://bugs.python.org/issue10789); so we'll just + # stick to the format that works in both (oddly the keyword argument + # works in py2.x but only with reentrant locks). + was_locked = lock.acquire(False) try: yield was_locked finally: From 6cb9a0cb13cb3f98bdbf573d7af9496471f8dc2d Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 2 Dec 2014 15:22:53 -0800 Subject: [PATCH 170/240] Add a simplistic hello world example Change-Id: I1d6e6535ab09d7f6c9d9ca3e2663983644b7a8a1 --- doc/source/examples.rst | 12 ++++ taskflow/examples/hello_world.py | 110 +++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 taskflow/examples/hello_world.py diff --git a/doc/source/examples.rst b/doc/source/examples.rst index 338c3921..5328da54 100644 --- a/doc/source/examples.rst +++ b/doc/source/examples.rst @@ -1,3 +1,15 @@ +Hello world +=========== + +.. note:: + + Full source located at :example:`hello_world`. + +.. literalinclude:: ../../taskflow/examples/hello_world.py + :language: python + :linenos: + :lines: 16- + Passing values from and to tasks ================================ diff --git a/taskflow/examples/hello_world.py b/taskflow/examples/hello_world.py new file mode 100644 index 00000000..22f6a3b0 --- /dev/null +++ b/taskflow/examples/hello_world.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import sys + +logging.basicConfig(level=logging.ERROR) + +top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, + os.pardir)) +sys.path.insert(0, top_dir) + +try: + import eventlet # noqa + EVENTLET_AVAILABLE = True +except ImportError: + EVENTLET_AVAILABLE = False + +from taskflow import engines +from taskflow.patterns import linear_flow as lf +from taskflow.patterns import unordered_flow as uf +from taskflow import task +from taskflow.types import futures + + +# INTRO: This is the defacto hello world equivalent for taskflow; it shows how +# a overly simplistic workflow can be created that runs using different +# engines using different styles of execution (all can be used to run in +# parallel if a workflow is provided that is parallelizable). + +class PrinterTask(task.Task): + def __init__(self, name, show_name=True, inject=None): + super(PrinterTask, self).__init__(name, inject=inject) + self._show_name = show_name + + def execute(self, output): + if self._show_name: + print("%s: %s" % (self.name, output)) + else: + print(output) + + +# This will be the work that we want done, which for this example is just to +# print 'hello world' (like a song) using different tasks and different +# execution models. +song = lf.Flow("beats") + +# Unordered flows when ran can be ran in parallel; and a chorus is everyone +# singing at once of course! +hi_chorus = uf.Flow('hello') +world_chorus = uf.Flow('world') +for (name, hello, world) in [('bob', 'hello', 'world'), + ('joe', 'hellooo', 'worllllld'), + ('sue', "helloooooo!", 'wooorllld!')]: + hi_chorus.add(PrinterTask("%s@hello" % name, + # This will show up to the execute() method of + # the task as the argument named 'output' (which + # will allow us to print the character we want). + inject={'output': hello})) + world_chorus.add(PrinterTask("%s@world" % name, + inject={'output': world})) + +# The composition starts with the conductor and then runs in sequence with +# the chorus running in parallel, but no matter what the 'hello' chorus must +# always run before the 'world' chorus (otherwise the world will fall apart). +song.add(PrinterTask("conductor@begin", + show_name=False, inject={'output': "*ding*"}), + hi_chorus, + world_chorus, + PrinterTask("conductor@end", + show_name=False, inject={'output': "*dong*"})) + +# Run in parallel using eventlet green threads... +if EVENTLET_AVAILABLE: + with futures.GreenThreadPoolExecutor() as executor: + e = engines.load(song, executor=executor, engine='parallel') + e.run() + + +# Run in parallel using real threads... +with futures.ThreadPoolExecutor(max_workers=1) as executor: + e = engines.load(song, executor=executor, engine='parallel') + e.run() + + +# Run in parallel using external processes... +with futures.ProcessPoolExecutor(max_workers=1) as executor: + e = engines.load(song, executor=executor, engine='parallel') + e.run() + + +# Run serially (aka, if the workflow could have been ran in parallel, it will +# not be when ran in this mode)... +e = engines.load(song, engine='serial') +e.run() From f862775c222e889f9945374413b002d8d50072e4 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 20 Dec 2014 11:18:24 -0800 Subject: [PATCH 171/240] Return the same namedtuple that the future module returns Instead of converting the namedtuple that the future wait function returns into a normal tuple, just have our own internal functions use the futures namedtuple directly instead and avoid any conversion to/from that namedtuple into a normal tuple. Change-Id: I54b2595af8d58db60843195034d66a623c20277c --- taskflow/utils/async_utils.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/taskflow/utils/async_utils.py b/taskflow/utils/async_utils.py index b055a27b..2fa3b5f6 100644 --- a/taskflow/utils/async_utils.py +++ b/taskflow/utils/async_utils.py @@ -54,8 +54,9 @@ def wait_for_any(fs, timeout=None): """ green_fs = sum(1 for f in fs if isinstance(f, futures.GreenFuture)) if not green_fs: - return tuple(_futures.wait(fs, timeout=timeout, - return_when=_futures.FIRST_COMPLETED)) + return _futures.wait(fs, + timeout=timeout, + return_when=_futures.FIRST_COMPLETED) else: non_green_fs = len(fs) - green_fs if non_green_fs: @@ -81,23 +82,24 @@ class _GreenWaiter(object): self.event.set() +def _partition_futures(fs): + done = set() + not_done = set() + for f in fs: + if f._state in _DONE_STATES: + done.add(f) + else: + not_done.add(f) + return done, not_done + + def _wait_for_any_green(fs, timeout=None): assert EVENTLET_AVAILABLE, 'eventlet is needed to wait on green futures' - def _partition_futures(fs): - done = set() - not_done = set() - for f in fs: - if f._state in _DONE_STATES: - done.add(f) - else: - not_done.add(f) - return (done, not_done) - with _base._AcquireFutures(fs): - (done, not_done) = _partition_futures(fs) + done, not_done = _partition_futures(fs) if done: - return (done, not_done) + return _base.DoneAndNotDoneFutures(done, not_done) waiter = _GreenWaiter() for f in fs: f._waiters.append(waiter) @@ -107,4 +109,5 @@ def _wait_for_any_green(fs, timeout=None): f._waiters.remove(waiter) with _base._AcquireFutures(fs): - return _partition_futures(fs) + done, not_done = _partition_futures(fs) + return _base.DoneAndNotDoneFutures(done, not_done) From f897008f5bf817895475e4e799c460082c736d6b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 21 Dec 2014 11:29:17 -0800 Subject: [PATCH 172/240] Ensure manager started/shutdown/joined and reset Ensure that when the task executor is started that we correctly create a new multiprocessing manager (if needed) and that on stop we correctly shut that manager down and join it. Also does a tiny adjustment to the joinable work item to move the finish logic into its own method and ensures that we have no targets on reset of the dispatcher. Change-Id: I688df323fb24a7e228f4fa237f2fa772d9c0dc62 --- taskflow/engines/action_engine/executor.py | 29 +++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index 5d063c5b..b1e969d0 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -17,6 +17,7 @@ import abc import functools import multiprocessing +from multiprocessing import managers import os import pickle import threading @@ -120,17 +121,20 @@ class _JoinedWorkItem(object): self._args = args self._kwargs = kwargs + def _on_finish(self): + w = timing.StopWatch() + w.start() + self._queue.join() + LOG.blather("Waited %0.2f seconds until task '%s' emitted" + " notifications were depleted", w.elapsed(), self._task) + def __call__(self): args = self._args kwargs = self._kwargs try: return self._func(self._task, *args, **kwargs) finally: - w = timing.StopWatch().start() - self._queue.join() - LOG.blather("Waited %0.2f seconds until task '%s' emitted" - " notifications were depleted", w.elapsed(), - self._task) + self._on_finish() class _EventSender(object): @@ -231,6 +235,8 @@ class _EventDispatcher(object): def reset(self): self._stop_when_empty = False + while self._targets: + self.deregister(self._targets.pop()) self._dead.clear() def interrupt(self): @@ -396,16 +402,25 @@ class ParallelProcessTaskExecutor(ParallelTaskExecutor): super(ParallelProcessTaskExecutor, self).__init__( executor=executor, max_workers=max_workers) self._manager = multiprocessing.Manager() - self._queue_factory = lambda: self._manager.JoinableQueue() self._dispatcher = _EventDispatcher( dispatch_periodicity=dispatch_periodicity) self._worker = None + def _queue_factory(self): + return self._manager.JoinableQueue() + def _create_executor(self, max_workers=None): return futures.ProcessPoolExecutor(max_workers=max_workers) def start(self): super(ParallelProcessTaskExecutor, self).start() + # TODO(harlowja): do something else here besides accessing a state + # of the manager internals (it doesn't seem to expose any way to know + # this information)... + if self._manager._state.value == managers.State.SHUTDOWN: + self._manager = multiprocessing.Manager() + if self._manager._state.value == managers.State.INITIAL: + self._manager.start() if not threading_utils.is_alive(self._worker): self._dispatcher.reset() self._worker = threading_utils.daemon_thread(self._dispatcher.run) @@ -418,6 +433,8 @@ class ParallelProcessTaskExecutor(ParallelTaskExecutor): self._worker.join() self._worker = None self._dispatcher.reset() + self._manager.shutdown() + self._manager.join() def _rebind_task(self, task, clone, queue, progress_callback=None): # Creates and binds proxies for all events the task could receive From fce7afbd8db26180816abe7d9566cf65277de376 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 17 Dec 2014 16:42:21 -0800 Subject: [PATCH 173/240] Remove less than useful action_engine __str__ The implementation of __str__ does not provide much useful information so instead just prefer the automatically provided one which provides equivalent information in a more well known format... Change-Id: I0a1683cfc22df1888a19f5af10d7a343462d3994 --- doc/source/notifications.rst | 12 ++++++------ taskflow/engines/action_engine/engine.py | 4 ---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/doc/source/notifications.rst b/doc/source/notifications.rst index 118b5aab..13c550ee 100644 --- a/doc/source/notifications.rst +++ b/doc/source/notifications.rst @@ -136,14 +136,14 @@ For example, this is how you can use >>> with printing.PrintingListener(eng): ... eng.run() ... - taskflow.engines.action_engine.engine.SerialActionEngine: ... has moved flow 'cat-dog' (...) into state 'RUNNING' - taskflow.engines.action_engine.engine.SerialActionEngine: ... has moved task 'CatTalk' (...) into state 'RUNNING' + has moved flow 'cat-dog' (...) into state 'RUNNING' + has moved task 'CatTalk' (...) into state 'RUNNING' meow - taskflow.engines.action_engine.engine.SerialActionEngine: ... has moved task 'CatTalk' (...) into state 'SUCCESS' with result 'cat' (failure=False) - taskflow.engines.action_engine.engine.SerialActionEngine: ... has moved task 'DogTalk' (...) into state 'RUNNING' + has moved task 'CatTalk' (...) into state 'SUCCESS' with result 'cat' (failure=False) + has moved task 'DogTalk' (...) into state 'RUNNING' woof - taskflow.engines.action_engine.engine.SerialActionEngine: ... has moved task 'DogTalk' (...) into state 'SUCCESS' with result 'dog' (failure=False) - taskflow.engines.action_engine.engine.SerialActionEngine: ... has moved flow 'cat-dog' (...) into state 'SUCCESS' + has moved task 'DogTalk' (...) into state 'SUCCESS' with result 'dog' (failure=False) + has moved flow 'cat-dog' (...) into state 'SUCCESS' Basic listener -------------- diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index f4158850..b0dbaa3a 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -20,7 +20,6 @@ import threading from concurrent import futures from oslo.utils import excutils -from oslo.utils import reflection from taskflow.engines.action_engine import compiler from taskflow.engines.action_engine import executor @@ -70,9 +69,6 @@ class ActionEngine(base.Engine): self._state_lock = threading.RLock() self._storage_ensured = False - def __str__(self): - return "%s: %s" % (reflection.get_class_name(self), id(self)) - def suspend(self): if not self._compiled: raise exc.InvalidState("Can not suspend an engine" From 1d84fdd09479ee05270073a0d6b4a0bf199c75ff Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 21 Dec 2014 16:23:10 -0800 Subject: [PATCH 174/240] Add edge labels for engine states The engine state diagram benefits slightly from having the event labels that cause transitions to other states so we might as well include it in the generated diagram. Change-Id: I733eba1d2dc6386c7b7ce8930fbfd41e29cdb602 --- doc/source/img/engine_states.svg | 6 +++--- tools/state_graph.py | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/doc/source/img/engine_states.svg b/doc/source/img/engine_states.svg index 079002ec..807b8ea5 100644 --- a/doc/source/img/engine_states.svg +++ b/doc/source/img/engine_states.svg @@ -1,8 +1,8 @@ - - -Engines statesGAME_OVERREVERTEDSUCCESSSUSPENDEDFAILUREUNDEFINEDRESUMINGSCHEDULINGANALYZINGWAITINGstart + +Engines statesGAME_OVERREVERTEDon revertedSUCCESSon successSUSPENDEDon suspendedFAILUREon failedUNDEFINEDRESUMINGon startSCHEDULINGon scheduleANALYZINGon finishedon scheduleWAITINGon waiton waiton analyzestart diff --git a/tools/state_graph.py b/tools/state_graph.py index c5d72d02..b4f9d53b 100755 --- a/tools/state_graph.py +++ b/tools/state_graph.py @@ -128,7 +128,7 @@ def main(): 'fontsize': '11', } nodes = {} - for (start_state, _on_event, end_state) in source: + for (start_state, on_event, end_state) in source: if start_state not in nodes: start_node_attrs = node_attrs.copy() text_color = map_color(internal_states, start_state) @@ -143,7 +143,20 @@ def main(): end_node_attrs['fontcolor'] = text_color nodes[end_state] = pydot.Node(end_state, **end_node_attrs) g.add_node(nodes[end_state]) - g.add_edge(pydot.Edge(nodes[start_state], nodes[end_state])) + if options.engines: + edge_attrs = { + 'label': "on %s" % on_event + } + if 'reverted' in on_event: + edge_attrs['fontcolor'] = 'darkorange' + if 'fail' in on_event: + edge_attrs['fontcolor'] = 'red' + if 'success' in on_event: + edge_attrs['fontcolor'] = 'green' + else: + edge_attrs = {} + g.add_edge(pydot.Edge(nodes[start_state], nodes[end_state], + **edge_attrs)) start = pydot.Node("__start__", shape="point", width="0.1", xlabel='start', fontcolor='green', **node_attrs) From d498fdb1f6aa9898facedb815751da0e912e3fe0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 25 Dec 2014 12:38:08 -0800 Subject: [PATCH 175/240] Register with 'ANY' in the cloned process Seems like we should have registered with 'ANY' since otherwise if a cloned task emits a unknown notification then the 'ANY' event should be triggered instead of not being emitted. Change-Id: Iedd3c0eb034043ba8e5b9e9a02a6e49c451e17b3 --- taskflow/engines/action_engine/executor.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index 5d063c5b..34142058 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -428,10 +428,6 @@ class ParallelProcessTaskExecutor(ParallelTaskExecutor): for (event_type, listeners) in task.notifier.listeners_iter(): if listeners: needed.add(event_type) - # We don't register for the 'ANY' event; since that meta event type - # will be correctly proxied by the task notifier directly without - # needing clone replication. - needed.discard(task.notifier.ANY) if progress_callback is not None: needed.add(_UPDATE_PROGRESS) for event_type in needed: From 05518caf9299ba487cfea76c80bfe3c2723d0647 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 27 Dec 2014 19:34:19 -0800 Subject: [PATCH 176/240] Update statement around stopwatch thread safety Watches are actually thread-safe as long as individual watch objects are not shared across threads or the operations on watches are protected by locks. Change-Id: I3565e7b76ec0866bcbca8666bcdf727441e01b10 --- taskflow/types/timing.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/taskflow/types/timing.py b/taskflow/types/timing.py index ab9a4c48..c7cb38db 100644 --- a/taskflow/types/timing.py +++ b/taskflow/types/timing.py @@ -49,7 +49,10 @@ class StopWatch(object): Inspired by: apache-commons-lang java stopwatch. - Not thread-safe. + Not thread-safe (when a single watch is mutated by multiple threads at + the same time). Thread-safe when used by a single thread (not shared) or + when operations are performed in a thread-safe manner on these objects by + wrapping those operations with locks. """ _STARTED = 'STARTED' _STOPPED = 'STOPPED' From 49ac8ec3ee7d02045a0faaa0ebf13a2a380a2a9e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 29 Dec 2014 09:32:21 -0800 Subject: [PATCH 177/240] Avoid creating a temporary list(s) for tree type Instead of creating a temporary list of the node using its __iter__() function and then reversing that list just use the natively provided reverse_iter() method instead that reduces this wasteful list copying and creating in the first place. Also does the same in the pformat() function which was needlessly creating a temporary list of children nodes instead of just using the nodes __iter__() functionality directly. Change-Id: Ice4001e6d014d2c0a1f7d8b916c60370fd5443a7 --- taskflow/types/tree.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/taskflow/types/tree.py b/taskflow/types/tree.py index 4f4f88d3..6eb960b4 100644 --- a/taskflow/types/tree.py +++ b/taskflow/types/tree.py @@ -45,8 +45,7 @@ class _DFSIter(object): # Visit the node. yield node # Traverse the left & right subtree. - for child_node in reversed(list(node)): - stack.append(child_node) + stack.extend(node.reverse_iter()) class Node(object): @@ -136,10 +135,10 @@ class Node(object): else: yield "__%s" % six.text_type(node.item) prefix = " " * 2 - children = list(node) - for (i, child) in enumerate(children): + child_count = node.child_count() + for (i, child) in enumerate(node): for (j, text) in enumerate(_inner_pformat(child, level + 1)): - if j == 0 or i + 1 < len(children): + if j == 0 or i + 1 < child_count: text = prefix + "|" + text else: text = prefix + " " + text From 50c5441efdd4406d3cb2dac40823a9c318175d9b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 25 Dec 2014 14:56:26 -0800 Subject: [PATCH 178/240] Use a single shared queue for an executors lifecycle Instead of having many queues (one per task) just have a shared one and use sender identifiers to know which originating task to proxy back to. This scales better and avoids the needless polling of many queues for a potential message to emit (and can now instead just poll one queue); when now we can just poll a single queue for any messages (this does though remove the ability to throttle messages from a given sender). Change-Id: I3566a5ab20ad15a80a4c6969f48b076ddde1d7ac --- taskflow/engines/action_engine/executor.py | 338 +++++++++++---------- 1 file changed, 183 insertions(+), 155 deletions(-) diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index b1e969d0..9548f072 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -15,18 +15,18 @@ # under the License. import abc -import functools +import collections import multiprocessing from multiprocessing import managers import os import pickle -import threading from oslo.utils import excutils +from oslo.utils import reflection from oslo.utils import timeutils +from oslo.utils import uuidutils import six from six.moves import queue as compat_queue -from six.moves import range as compat_range from taskflow import logging from taskflow import task as task_atom @@ -52,18 +52,13 @@ _PICKLE_ERRORS = tuple(_PICKLE_ERRORS) _SEND_ERRORS = (IOError, EOFError) _UPDATE_PROGRESS = task_atom.EVENT_UPDATE_PROGRESS +# Message types/kind sent from worker/child processes... +_KIND_COMPLETE_ME = 'complete_me' +_KIND_EVENT = 'event' + LOG = logging.getLogger(__name__) -def _maybe_forever(limit=None): - if limit is None: - while True: - yield - else: - for i in compat_range(0, limit): - yield - - def _execute_task(task, arguments, progress_callback=None): with notifier.register_deregister(task.notifier, _UPDATE_PROGRESS, @@ -99,12 +94,52 @@ def _revert_task(task, arguments, result, failures, progress_callback=None): return (REVERTED, result) -class _JoinedWorkItem(object): +class _Channel(object): + """Helper wrapper around a multiprocessing queue used by a worker.""" + + def __init__(self, queue, identity): + self._queue = queue + self._identity = identity + self._sent_messages = collections.defaultdict(int) + self._pid = None + + @property + def sent_messages(self): + return self._sent_messages + + def put(self, message): + # NOTE(harlowja): this is done in late in execution to ensure that this + # happens in the child process and not the parent process (where the + # constructor is called). + if self._pid is None: + self._pid = os.getpid() + message.update({ + 'sent_on': timeutils.utcnow(), + 'sender': { + 'pid': self._pid, + 'id': self._identity, + }, + }) + if 'body' not in message: + message['body'] = {} + try: + self._queue.put(message) + except _PICKLE_ERRORS: + LOG.warn("Failed serializing message %s", message, exc_info=True) + return False + except _SEND_ERRORS: + LOG.warn("Failed sending message %s", message, exc_info=True) + return False + else: + self._sent_messages[message['kind']] += 1 + return True + + +class _WaitWorkItem(object): """The piece of work that will executed by a process executor. - This will call the target function, then wait until the queues items - have been completed (via calls to task_done) before offically being - finished. + This will call the target function, then wait until the tasks emitted + events/items have been depleted before offically being finished. NOTE(harlowja): this is done so that the task function will *not* return until all of its notifications have been proxied back to its originating @@ -114,19 +149,28 @@ class _JoinedWorkItem(object): previously finished task... """ - def __init__(self, queue, func, task, *args, **kwargs): - self._queue = queue + def __init__(self, channel, barrier, + func, task, *args, **kwargs): + self._channel = channel + self._barrier = barrier self._func = func self._task = task self._args = args self._kwargs = kwargs def _on_finish(self): - w = timing.StopWatch() - w.start() - self._queue.join() - LOG.blather("Waited %0.2f seconds until task '%s' emitted" - " notifications were depleted", w.elapsed(), self._task) + sent_events = self._channel.sent_messages.get(_KIND_EVENT, 0) + if sent_events: + message = { + 'created_on': timeutils.utcnow(), + 'kind': _KIND_COMPLETE_ME, + } + if self._channel.put(message): + w = timing.StopWatch().start() + self._barrier.wait() + LOG.blather("Waited %s seconds until task '%s' %s emitted" + " notifications were depleted", w.elapsed(), + self._task, sent_events) def __call__(self): args = self._args @@ -140,57 +184,44 @@ class _JoinedWorkItem(object): class _EventSender(object): """Sends event information from a child worker process to its creator.""" - def __init__(self, queue): - self._queue = queue - self._pid = None + def __init__(self, channel): + self._channel = channel def __call__(self, event_type, details): - # NOTE(harlowja): this is done in late in execution to ensure that this - # happens in the child process and not the parent process (where the - # constructor is called). - if self._pid is None: - self._pid = os.getpid() message = { 'created_on': timeutils.utcnow(), - 'sender': { - 'pid': self._pid, - }, + 'kind': _KIND_EVENT, 'body': { 'event_type': event_type, 'details': details, }, } - try: - self._queue.put(message) - except _PICKLE_ERRORS: - LOG.warn("Failed serializing message %s", message, exc_info=True) - except _SEND_ERRORS: - LOG.warn("Failed sending message %s", message, exc_info=True) + self._channel.put(message) -class _EventTarget(object): - """An immutable helper object that represents a target of an event.""" +class _Target(object): + """An immutable helper object that represents a target of a message.""" - def __init__(self, future, task, queue): - self.future = future + def __init__(self, task, barrier, identity): self.task = task - self.queue = queue + self.barrier = barrier + self.identity = identity + # Counters used to track how many message 'kinds' were proxied... + self.dispatched = collections.defaultdict(int) + + def __repr__(self): + return "<%s at 0x%x targeting '%s' with identity '%s'>" % ( + reflection.get_class_name(self), id(self), + self.task, self.identity) -class _EventDispatcher(object): - """Dispatches event information received from child worker processes.""" +class _Dispatcher(object): + """Dispatches messages received from child worker processes.""" # When the run() method is busy (typically in a thread) we want to set # these so that the thread can know how long to sleep when there is no - # active work to dispatch (when there is active targets, there queues - # will have amount/count of items removed before returning to work on - # the next target...) + # active work to dispatch. _SPIN_PERIODICITY = 0.01 - _SPIN_DISPATCH_AMOUNT = 1 - - # TODO(harlowja): look again at using a non-polling mechanism that uses - # select instead of queues to achieve better ability to detect when - # messages are ready/available... def __init__(self, dispatch_periodicity=None): if dispatch_periodicity is None: @@ -198,83 +229,84 @@ class _EventDispatcher(object): if dispatch_periodicity <= 0: raise ValueError("Provided dispatch periodicity must be greater" " than zero and not '%s'" % dispatch_periodicity) - self._targets = set() + self._targets = {} self._dead = threading_utils.Event() - self._lock = threading.Lock() - self._periodicity = dispatch_periodicity + self._dispatch_periodicity = dispatch_periodicity self._stop_when_empty = False - def register(self, target): - with self._lock: - self._targets.add(target) + def register(self, identity, target): + self._targets[identity] = target - def _dispatch_until_empty(self, target, limit=None): - it = _maybe_forever(limit=limit) - while True: - try: - six.next(it) - except StopIteration: - break - else: - try: - message = target.queue.get_nowait() - except compat_queue.Empty: - break - else: - try: - self._dispatch(target.task, message) - finally: - target.queue.task_done() - - def deregister(self, target): - with self._lock: - try: - self._targets.remove(target) - except KeyError: - pass + def deregister(self, identity): + try: + target = self._targets.pop(identity) + except KeyError: + pass + else: + # Just incase set the barrier to unblock any worker... + target.barrier.set() + if LOG.isEnabledFor(logging.BLATHER): + LOG.blather("Dispatched %s messages %s to target '%s' during" + " the lifetime of its existence in the dispatcher", + sum(six.itervalues(target.dispatched)), + dict(target.dispatched), target) def reset(self): self._stop_when_empty = False - while self._targets: - self.deregister(self._targets.pop()) self._dead.clear() + if self._targets: + leftover = set(six.iterkeys(self._targets)) + while leftover: + self.deregister(leftover.pop()) def interrupt(self): self._stop_when_empty = True self._dead.set() - def _dispatch(self, task, message): - LOG.blather("Dispatching message %s to task '%s'", message, task) - body = message['body'] - task.notifier.notify(body['event_type'], body['details']) + def _dispatch(self, message): + if LOG.isEnabledFor(logging.BLATHER): + LOG.blather("Dispatching message %s (it took %s seconds" + " for it to arrive for processing after being" + " sent)", message, + timeutils.delta_seconds(message['sent_on'], + timeutils.utcnow())) + try: + kind = message['kind'] + sender = message['sender'] + body = message['body'] + except (KeyError, ValueError, TypeError): + LOG.warn("Badly formatted message %s received", message, + exc_info=True) + return + target = self._targets.get(sender['id']) + if target is None: + # Must of been removed... + return + if kind == _KIND_COMPLETE_ME: + target.dispatched[kind] += 1 + target.barrier.set() + elif kind == _KIND_EVENT: + task = target.task + target.dispatched[kind] += 1 + task.notifier.notify(body['event_type'], body['details']) + else: + LOG.warn("Unknown message '%s' found in message from sender" + " %s to target '%s'", kind, sender, target) - def _dispatch_iter(self, targets): - # A generator that yields at certain points to allow the main run() - # method to use this to dispatch in iterations (and also allows it - # to check if it has been stopped between iterations). - for target in targets: - if target not in self._targets: - # Must of been removed... - continue - # NOTE(harlowja): Limits are used here to avoid one - # task unequally dispatching, this forces round-robin - # like behavior... - self._dispatch_until_empty(target, - limit=self._SPIN_DISPATCH_AMOUNT) - yield target - - def run(self): - w = timing.StopWatch(duration=self._periodicity) + def run(self, queue): + w = timing.StopWatch(duration=self._dispatch_periodicity) while (not self._dead.is_set() or (self._stop_when_empty and self._targets)): w.restart() - with self._lock: - targets = self._targets.copy() - for _target in self._dispatch_iter(targets): - if self._stop_when_empty: - continue - if self._dead.is_set(): + leftover = w.leftover() + while leftover: + try: + message = queue.get(timeout=leftover) + except compat_queue.Empty: break + else: + self._dispatch(message) + leftover = w.leftover() leftover = w.leftover() if leftover: self._dead.wait(leftover) @@ -402,12 +434,11 @@ class ParallelProcessTaskExecutor(ParallelTaskExecutor): super(ParallelProcessTaskExecutor, self).__init__( executor=executor, max_workers=max_workers) self._manager = multiprocessing.Manager() - self._dispatcher = _EventDispatcher( + self._dispatcher = _Dispatcher( dispatch_periodicity=dispatch_periodicity) + # Only created after starting... self._worker = None - - def _queue_factory(self): - return self._manager.JoinableQueue() + self._queue = None def _create_executor(self, max_workers=None): return futures.ProcessPoolExecutor(max_workers=max_workers) @@ -423,7 +454,9 @@ class ParallelProcessTaskExecutor(ParallelTaskExecutor): self._manager.start() if not threading_utils.is_alive(self._worker): self._dispatcher.reset() - self._worker = threading_utils.daemon_thread(self._dispatcher.run) + self._queue = self._manager.Queue() + self._worker = threading_utils.daemon_thread(self._dispatcher.run, + self._queue) self._worker.start() def stop(self): @@ -432,11 +465,12 @@ class ParallelProcessTaskExecutor(ParallelTaskExecutor): if threading_utils.is_alive(self._worker): self._worker.join() self._worker = None + self._queue = None self._dispatcher.reset() self._manager.shutdown() self._manager.join() - def _rebind_task(self, task, clone, queue, progress_callback=None): + def _rebind_task(self, task, clone, channel, progress_callback=None): # Creates and binds proxies for all events the task could receive # so that when the clone runs in another process that this task # can recieve the same notifications (thus making it look like the @@ -452,8 +486,7 @@ class ParallelProcessTaskExecutor(ParallelTaskExecutor): if progress_callback is not None: needed.add(_UPDATE_PROGRESS) for event_type in needed: - clone.notifier.register(event_type, _EventSender(queue)) - return needed + clone.notifier.register(event_type, _EventSender(channel)) def _submit_task(self, func, task, *args, **kwargs): """Submit a function to run the given task (with given args/kwargs). @@ -478,49 +511,44 @@ class ParallelProcessTaskExecutor(ParallelTaskExecutor): any listeners) and then reattach a new set of listeners that will now instead of calling the desired listeners just place messages for this process (a dispatcher thread that is created in this class) - to dispatch to the original task (using a per task queue that is used - and associated to know which task to proxy back too, since it is - possible that there many be *many* subprocess running at the same - time, each running a different task). + to dispatch to the original task (using a common queue + per task + sender identity/target that is used and associated to know which task + to proxy back too, since it is possible that there many be *many* + subprocess running at the same time, each running a different task + and using the same common queue to submit messages back to). Once the subprocess task has finished execution, the executor will - then trigger a callback (``on_done`` in this case) that will remove - the task + queue from the dispatcher (which will stop any further - proxying back to the original task). + then trigger a callback that will remove the task + target from the + dispatcher (which will stop any further proxying back to the original + task). """ progress_callback = kwargs.pop('progress_callback', None) clone = task.copy(retain_listeners=False) - queue = self._queue_factory() - bound = self._rebind_task(task, clone, queue, - progress_callback=progress_callback) - LOG.blather("Bound %s event types to clone of '%s'", bound, task) - if progress_callback is not None: - binder = functools.partial(task.notifier.register, - _UPDATE_PROGRESS, progress_callback) - unbinder = functools.partial(task.notifier.deregister, - _UPDATE_PROGRESS, progress_callback) - else: - binder = unbinder = lambda: None + identity = uuidutils.generate_uuid() + target = _Target(task, self._manager.Event(), identity) + channel = _Channel(self._queue, identity) + self._rebind_task(task, clone, channel, + progress_callback=progress_callback) - # Ensure the target task (not the clone) is ready and able to receive - # dispatched messages (and start the dispatching process by - # registering) with the dispatcher. - binder() - work = _JoinedWorkItem(queue, func, clone, *args, **kwargs) + def register(): + if progress_callback is not None: + task.notifier.register(_UPDATE_PROGRESS, progress_callback) + self._dispatcher.register(identity, target) + + def deregister(): + if progress_callback is not None: + task.notifier.deregister(_UPDATE_PROGRESS, progress_callback) + self._dispatcher.deregister(identity) + + register() + work = _WaitWorkItem(channel, target.barrier, + func, clone, *args, **kwargs) try: fut = self._executor.submit(work) except RuntimeError: with excutils.save_and_reraise_exception(): - unbinder() - - # This will trigger the proxying to begin... - target = _EventTarget(fut, task, queue) - self._dispatcher.register(target) - - def on_done(unbinder, target, fut): - self._dispatcher.deregister(target) - unbinder() + deregister() fut.atom = task - fut.add_done_callback(functools.partial(on_done, unbinder, target)) + fut.add_done_callback(lambda fut: deregister()) return fut From 2ed1ad94904a314200164835541b9870ce6289e4 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 30 Dec 2014 10:29:55 -0800 Subject: [PATCH 179/240] Disallowing starting the executor when worker running To avoid consistency/threading/runtime issues stop the action engine executor from being started if it is already running with a valid worker thread. Change-Id: I39925e55e7b171f289152d941ebdf390552f880c --- taskflow/engines/action_engine/executor.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index 1ecd1adc..d110c313 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -444,6 +444,9 @@ class ParallelProcessTaskExecutor(ParallelTaskExecutor): return futures.ProcessPoolExecutor(max_workers=max_workers) def start(self): + if threading_utils.is_alive(self._worker): + raise RuntimeError("Worker thread must be stopped via stop()" + " before starting/restarting") super(ParallelProcessTaskExecutor, self).start() # TODO(harlowja): do something else here besides accessing a state # of the manager internals (it doesn't seem to expose any way to know @@ -452,12 +455,11 @@ class ParallelProcessTaskExecutor(ParallelTaskExecutor): self._manager = multiprocessing.Manager() if self._manager._state.value == managers.State.INITIAL: self._manager.start() - if not threading_utils.is_alive(self._worker): - self._dispatcher.reset() - self._queue = self._manager.Queue() - self._worker = threading_utils.daemon_thread(self._dispatcher.run, - self._queue) - self._worker.start() + self._dispatcher.reset() + self._queue = self._manager.Queue() + self._worker = threading_utils.daemon_thread(self._dispatcher.run, + self._queue) + self._worker.start() def stop(self): self._dispatcher.interrupt() From b52db18522a938a995039790649821b65c26ef67 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 23 Dec 2014 15:33:13 -0800 Subject: [PATCH 180/240] Allow specifying the engine 'executor' as a string To enable a parallel process executor to be used without having to pass in a futures executor allow for the executor option to be a string 'processes' (or similar) that will cause the default parallel process executor to be used automatically. Also allow for a 'threads' string that ensure a parallel thread executor is used to match the ability to uses processes. This also adjusts the WBE engine to have a similar executor fetching function (which in the WBE case now validates a provided executor to be of the desired type). Change-Id: I54a82584c32c697922507b4f6e01ea7b8acc73c6 --- taskflow/engines/action_engine/engine.py | 121 ++++++++++++++---- taskflow/engines/action_engine/executor.py | 4 + taskflow/engines/worker_based/engine.py | 36 ++++-- .../tests/unit/action_engine/test_creation.py | 80 ++++++++++++ .../{test_engine.py => test_creation.py} | 66 ++++++---- 5 files changed, 246 insertions(+), 61 deletions(-) create mode 100644 taskflow/tests/unit/action_engine/test_creation.py rename taskflow/tests/unit/worker_based/{test_engine.py => test_creation.py} (50%) diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index b0dbaa3a..7ececb9f 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -14,12 +14,13 @@ # License for the specific language governing permissions and limitations # under the License. -import abc +import collections import contextlib import threading from concurrent import futures from oslo.utils import excutils +import six from taskflow.engines.action_engine import compiler from taskflow.engines.action_engine import executor @@ -199,11 +200,6 @@ class ActionEngine(base.Engine): self._runtime.reset_all() self._change_state(states.PENDING) - @abc.abstractproperty - def _task_executor(self): - return self._task_executor_factory() - pass - @misc.cachedproperty def _compiler(self): return self._compiler_factory(self._flow) @@ -224,28 +220,105 @@ class SerialActionEngine(ActionEngine): """Engine that runs tasks in serial manner.""" _storage_factory = atom_storage.SingleThreadedStorage - @misc.cachedproperty - def _task_executor(self): - return executor.SerialTaskExecutor() + def __init__(self, flow, flow_detail, backend, options): + super(SerialActionEngine, self).__init__(flow, flow_detail, + backend, options) + self._task_executor = executor.SerialTaskExecutor() + + +class _ExecutorTypeMatch(collections.namedtuple('_ExecutorTypeMatch', + ['types', 'executor_cls'])): + def matches(self, executor): + return isinstance(executor, self.types) + + +class _ExecutorTextMatch(collections.namedtuple('_ExecutorTextMatch', + ['strings', 'executor_cls'])): + def matches(self, text): + return text.lower() in self.strings class ParallelActionEngine(ActionEngine): """Engine that runs tasks in parallel manner.""" _storage_factory = atom_storage.MultiThreadedStorage - @misc.cachedproperty - def _task_executor(self): - kwargs = { - 'executor': self._options.get('executor'), - 'max_workers': self._options.get('max_workers'), - } - # The reason we use the library/built-in futures is to allow for - # instances of that to be detected and handled correctly, instead of - # forcing everyone to use our derivatives... - if isinstance(kwargs['executor'], futures.ProcessPoolExecutor): - executor_cls = executor.ParallelProcessTaskExecutor - kwargs['dispatch_periodicity'] = self._options.get( - 'dispatch_periodicity') - else: - executor_cls = executor.ParallelThreadTaskExecutor + # One of these types should match when a object (non-string) is provided + # for the 'executor' option. + # + # NOTE(harlowja): the reason we use the library/built-in futures is to + # allow for instances of that to be detected and handled correctly, instead + # of forcing everyone to use our derivatives... + _executor_cls_matchers = [ + _ExecutorTypeMatch((futures.ThreadPoolExecutor,), + executor.ParallelThreadTaskExecutor), + _ExecutorTypeMatch((futures.ProcessPoolExecutor,), + executor.ParallelProcessTaskExecutor), + _ExecutorTypeMatch((futures.Executor,), + executor.ParallelThreadTaskExecutor), + ] + + # One of these should match when a string/text is provided for the + # 'executor' option (a mixed case equivalent is allowed since the match + # will be lower-cased before checking). + _executor_str_matchers = [ + _ExecutorTextMatch(frozenset(['processes', 'process']), + executor.ParallelProcessTaskExecutor), + _ExecutorTextMatch(frozenset(['thread', 'threads', 'threaded']), + executor.ParallelThreadTaskExecutor), + ] + + # Used when no executor is provided (either a string or object)... + _default_executor_cls = executor.ParallelThreadTaskExecutor + + def __init__(self, flow, flow_detail, backend, options): + super(ParallelActionEngine, self).__init__(flow, flow_detail, + backend, options) + # This ensures that any provided executor will be validated before + # we get to far in the compilation/execution pipeline... + self._task_executor = self._fetch_task_executor(self._options) + + @classmethod + def _fetch_task_executor(cls, options): + kwargs = {} + executor_cls = cls._default_executor_cls + # Match the desired executor to a class that will work with it... + desired_executor = options.get('executor') + if isinstance(desired_executor, six.string_types): + matched_executor_cls = None + for m in cls._executor_str_matchers: + if m.matches(desired_executor): + matched_executor_cls = m.executor_cls + break + if matched_executor_cls is None: + expected = set() + for m in cls._executor_str_matchers: + expected.update(m.strings) + raise ValueError("Unknown executor string '%s' expected" + " one of %s (or mixed case equivalent)" + % (desired_executor, list(expected))) + else: + executor_cls = matched_executor_cls + elif desired_executor is not None: + matched_executor_cls = None + for m in cls._executor_cls_matchers: + if m.matches(desired_executor): + matched_executor_cls = m.executor_cls + break + if matched_executor_cls is None: + expected = set() + for m in cls._executor_cls_matchers: + expected.update(m.types) + raise TypeError("Unknown executor type '%s' expected an" + " instance of %s" % (type(desired_executor), + list(expected))) + else: + executor_cls = matched_executor_cls + kwargs['executor'] = desired_executor + for k in getattr(executor_cls, 'OPTIONS', []): + if k == 'executor': + continue + try: + kwargs[k] = options[k] + except KeyError: + pass return executor_cls(**kwargs) diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index 1ecd1adc..6ffbc0f2 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -373,6 +373,8 @@ class ParallelTaskExecutor(TaskExecutor): to concurrent.Futures.Executor. """ + OPTIONS = frozenset(['max_workers']) + def __init__(self, executor=None, max_workers=None): self._executor = executor self._max_workers = max_workers @@ -429,6 +431,8 @@ class ParallelProcessTaskExecutor(ParallelTaskExecutor): the parent are executed on events in the child. """ + OPTIONS = frozenset(['max_workers', 'dispatch_periodicity']) + def __init__(self, executor=None, max_workers=None, dispatch_periodicity=None): super(ParallelProcessTaskExecutor, self).__init__( diff --git a/taskflow/engines/worker_based/engine.py b/taskflow/engines/worker_based/engine.py index df915fc9..8011222c 100644 --- a/taskflow/engines/worker_based/engine.py +++ b/taskflow/engines/worker_based/engine.py @@ -18,7 +18,6 @@ from taskflow.engines.action_engine import engine from taskflow.engines.worker_based import executor from taskflow.engines.worker_based import protocol as pr from taskflow import storage as t_storage -from taskflow.utils import misc class WorkerBasedActionEngine(engine.ActionEngine): @@ -45,17 +44,30 @@ class WorkerBasedActionEngine(engine.ActionEngine): _storage_factory = t_storage.SingleThreadedStorage - @misc.cachedproperty - def _task_executor(self): + def __init__(self, flow, flow_detail, backend, options): + super(WorkerBasedActionEngine, self).__init__(flow, flow_detail, + backend, options) + # This ensures that any provided executor will be validated before + # we get to far in the compilation/execution pipeline... + self._task_executor = self._fetch_task_executor(self._options, + self._flow_detail) + + @classmethod + def _fetch_task_executor(cls, options, flow_detail): try: - return self._options['executor'] + e = options['executor'] + if not isinstance(e, executor.WorkerTaskExecutor): + raise TypeError("Expected an instance of type '%s' instead of" + " type '%s' for 'executor' option" + % (executor.WorkerTaskExecutor, type(e))) + return e except KeyError: return executor.WorkerTaskExecutor( - uuid=self._flow_detail.uuid, - url=self._options.get('url'), - exchange=self._options.get('exchange', 'default'), - topics=self._options.get('topics', []), - transport=self._options.get('transport'), - transport_options=self._options.get('transport_options'), - transition_timeout=self._options.get('transition_timeout', - pr.REQUEST_TIMEOUT)) + uuid=flow_detail.uuid, + url=options.get('url'), + exchange=options.get('exchange', 'default'), + topics=options.get('topics', []), + transport=options.get('transport'), + transport_options=options.get('transport_options'), + transition_timeout=options.get('transition_timeout', + pr.REQUEST_TIMEOUT)) diff --git a/taskflow/tests/unit/action_engine/test_creation.py b/taskflow/tests/unit/action_engine/test_creation.py new file mode 100644 index 00000000..c8a0b436 --- /dev/null +++ b/taskflow/tests/unit/action_engine/test_creation.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import testtools + +from taskflow.engines.action_engine import engine +from taskflow.engines.action_engine import executor +from taskflow.patterns import linear_flow as lf +from taskflow.persistence import backends +from taskflow import test +from taskflow.tests import utils +from taskflow.types import futures as futures +from taskflow.utils import async_utils as au +from taskflow.utils import persistence_utils as pu + + +class ParallelCreationTest(test.TestCase): + @staticmethod + def _create_engine(**kwargs): + flow = lf.Flow('test-flow').add(utils.DummyTask()) + backend = backends.fetch({'connection': 'memory'}) + flow_detail = pu.create_flow_detail(flow, backend=backend) + options = kwargs.copy() + return engine.ParallelActionEngine(flow, flow_detail, + backend, options) + + def test_thread_string_creation(self): + for s in ['threads', 'threaded', 'thread']: + eng = self._create_engine(executor=s) + self.assertIsInstance(eng._task_executor, + executor.ParallelThreadTaskExecutor) + + def test_process_string_creation(self): + for s in ['process', 'processes']: + eng = self._create_engine(executor=s) + self.assertIsInstance(eng._task_executor, + executor.ParallelProcessTaskExecutor) + + def test_thread_executor_creation(self): + with futures.ThreadPoolExecutor(1) as e: + eng = self._create_engine(executor=e) + self.assertIsInstance(eng._task_executor, + executor.ParallelThreadTaskExecutor) + + def test_process_executor_creation(self): + with futures.ProcessPoolExecutor(1) as e: + eng = self._create_engine(executor=e) + self.assertIsInstance(eng._task_executor, + executor.ParallelProcessTaskExecutor) + + @testtools.skipIf(not au.EVENTLET_AVAILABLE, 'eventlet is not available') + def test_green_executor_creation(self): + with futures.GreenThreadPoolExecutor(1) as e: + eng = self._create_engine(executor=e) + self.assertIsInstance(eng._task_executor, + executor.ParallelThreadTaskExecutor) + + def test_sync_executor_creation(self): + with futures.SynchronousExecutor() as e: + eng = self._create_engine(executor=e) + self.assertIsInstance(eng._task_executor, + executor.ParallelThreadTaskExecutor) + + def test_invalid_creation(self): + self.assertRaises(ValueError, self._create_engine, executor='crap') + self.assertRaises(TypeError, self._create_engine, executor=2) + self.assertRaises(TypeError, self._create_engine, executor=object()) diff --git a/taskflow/tests/unit/worker_based/test_engine.py b/taskflow/tests/unit/worker_based/test_creation.py similarity index 50% rename from taskflow/tests/unit/worker_based/test_engine.py rename to taskflow/tests/unit/worker_based/test_creation.py index f274a829..6764926a 100644 --- a/taskflow/tests/unit/worker_based/test_engine.py +++ b/taskflow/tests/unit/worker_based/test_creation.py @@ -15,7 +15,9 @@ # under the License. from taskflow.engines.worker_based import engine +from taskflow.engines.worker_based import executor from taskflow.patterns import linear_flow as lf +from taskflow.persistence import backends from taskflow import test from taskflow.test import mock from taskflow.tests import utils @@ -23,24 +25,25 @@ from taskflow.utils import persistence_utils as pu class TestWorkerBasedActionEngine(test.MockTestCase): + @staticmethod + def _create_engine(**kwargs): + flow = lf.Flow('test-flow').add(utils.DummyTask()) + backend = backends.fetch({'connection': 'memory'}) + flow_detail = pu.create_flow_detail(flow, backend=backend) + options = kwargs.copy() + return engine.WorkerBasedActionEngine(flow, flow_detail, + backend, options) - def setUp(self): - super(TestWorkerBasedActionEngine, self).setUp() - self.broker_url = 'test-url' - self.exchange = 'test-exchange' - self.topics = ['test-topic1', 'test-topic2'] - - # patch classes - self.executor_mock, self.executor_inst_mock = self.patchClass( + def _patch_in_executor(self): + executor_mock, executor_inst_mock = self.patchClass( engine.executor, 'WorkerTaskExecutor', attach_as='executor') + return executor_mock, executor_inst_mock def test_creation_default(self): - flow = lf.Flow('test-flow').add(utils.DummyTask()) - _, flow_detail = pu.temporary_flow_detail() - engine.WorkerBasedActionEngine(flow, flow_detail, None, {}).compile() - + executor_mock, executor_inst_mock = self._patch_in_executor() + eng = self._create_engine() expected_calls = [ - mock.call.executor_class(uuid=flow_detail.uuid, + mock.call.executor_class(uuid=eng.storage.flow_uuid, url=None, exchange='default', topics=[], @@ -51,21 +54,34 @@ class TestWorkerBasedActionEngine(test.MockTestCase): self.assertEqual(self.master_mock.mock_calls, expected_calls) def test_creation_custom(self): - flow = lf.Flow('test-flow').add(utils.DummyTask()) - _, flow_detail = pu.temporary_flow_detail() - config = {'url': self.broker_url, 'exchange': self.exchange, - 'topics': self.topics, 'transport': 'memory', - 'transport_options': {}, 'transition_timeout': 200} - engine.WorkerBasedActionEngine( - flow, flow_detail, None, config).compile() - + executor_mock, executor_inst_mock = self._patch_in_executor() + topics = ['test-topic1', 'test-topic2'] + exchange = 'test-exchange' + broker_url = 'test-url' + eng = self._create_engine( + url=broker_url, + exchange=exchange, + transport='memory', + transport_options={}, + transition_timeout=200, + topics=topics) expected_calls = [ - mock.call.executor_class(uuid=flow_detail.uuid, - url=self.broker_url, - exchange=self.exchange, - topics=self.topics, + mock.call.executor_class(uuid=eng.storage.flow_uuid, + url=broker_url, + exchange=exchange, + topics=topics, transport='memory', transport_options={}, transition_timeout=200) ] self.assertEqual(self.master_mock.mock_calls, expected_calls) + + def test_creation_custom_executor(self): + ex = executor.WorkerTaskExecutor('a', 'test-exchange', ['test-topic']) + eng = self._create_engine(executor=ex) + self.assertIs(eng._task_executor, ex) + self.assertIsInstance(eng._task_executor, executor.WorkerTaskExecutor) + + def test_creation_invalid_custom_executor(self): + self.assertRaises(TypeError, self._create_engine, executor=2) + self.assertRaises(TypeError, self._create_engine, executor='blah') From 69449ae301fd411722021b47197d0a7eda335cf7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 5 Jan 2015 15:05:00 -0800 Subject: [PATCH 181/240] Remove need to inherit/adjust netutils split The code we had for adjusting the netutils urlsplit function to add in a params method/property is no longer needed as that functionality is now pushed into the oslo.utils repo/package where it can be maintained there in a more proper manner instead; so we can now remove our adjustment code and just use the upstream code instead. Change-Id: I5ca05c0ac6a6221157a737ba20814cfd63adf51e --- taskflow/tests/unit/test_utils.py | 2 +- taskflow/utils/misc.py | 24 ++---------------------- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/taskflow/tests/unit/test_utils.py b/taskflow/tests/unit/test_utils.py index c69b769d..ba71cca2 100644 --- a/taskflow/tests/unit/test_utils.py +++ b/taskflow/tests/unit/test_utils.py @@ -140,7 +140,7 @@ class UriParseTest(test.TestCase): self.assertEqual('192.168.0.1', parsed.hostname) self.assertEqual('', parsed.fragment) self.assertEqual('/a/b/', parsed.path) - self.assertEqual({'c': 'd'}, parsed.params) + self.assertEqual({'c': 'd'}, parsed.params()) def test_port_provided(self): url = "rabbitmq://www.yahoo.com:5672" diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index a6b04fa7..dd3610e7 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -32,7 +32,6 @@ from oslo.utils import reflection import six from six.moves import map as compat_map from six.moves import range as compat_range -from six.moves.urllib import parse as urlparse from taskflow.types import failure from taskflow.types import notifier @@ -46,22 +45,6 @@ NUMERIC_TYPES = six.integer_types + (float,) _SCHEME_REGEX = re.compile(r"^([A-Za-z][A-Za-z0-9+.-]*):") -# FIXME(harlowja): This should be removed with the next version of oslo.utils -# which now has this functionality built-in, until then we are deriving from -# there base class and adding this functionality on... -# -# The change was merged @ https://review.openstack.org/#/c/118881/ -class ModifiedSplitResult(netutils._ModifiedSplitResult): - """A split result that exposes the query parameters as a dictionary.""" - - @property - def params(self): - if self.query: - return dict(urlparse.parse_qsl(self.query)) - else: - return {} - - def merge_uri(uri, conf): """Merges a parsed uri into the given configuration dictionary. @@ -80,7 +63,7 @@ def merge_uri(uri, conf): if uri.port is not None: hostname += ":%s" % (uri.port) conf.setdefault('hostname', hostname) - for (k, v) in six.iteritems(uri.params): + for (k, v) in six.iteritems(uri.params()): conf.setdefault(k, v) return conf @@ -140,10 +123,7 @@ def parse_uri(uri): if not match: raise ValueError("Uri %r does not start with a RFC 3986 compliant" " scheme" % (uri)) - split = netutils.urlsplit(uri) - return ModifiedSplitResult(scheme=split.scheme, fragment=split.fragment, - path=split.path, netloc=split.netloc, - query=split.query) + return netutils.urlsplit(uri) def clamp(value, minimum, maximum, on_clamped=None): From a18a939aae1dff42a4741a1852829abe5280aa80 Mon Sep 17 00:00:00 2001 From: Min Pae Date: Wed, 7 Jan 2015 12:26:19 -0800 Subject: [PATCH 182/240] Fix for job consumption example using wrong object Change job consumption example in Job documentation to use persistence.get_connection().destroy_logbook() instead of persistence.destroy_logbook() Change-Id: Ia6d0f2be4dd7cd161ef7b6d8afc32aab532fd69b Closes-Bug: 1408434 --- doc/source/jobs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/jobs.rst b/doc/source/jobs.rst index 63479279..f36d69a1 100644 --- a/doc/source/jobs.rst +++ b/doc/source/jobs.rst @@ -164,7 +164,7 @@ might look like: else: # I finished it, now cleanup. board.consume(my_job) - persistence.destroy_logbook(my_job.book.uuid) + persistence.get_connection().destroy_logbook(my_job.book.uuid) time.sleep(coffee_break_time) ... From 316b453558d157d8c1b07b015d6e90d08b0b8c3c Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Fri, 9 Jan 2015 18:36:27 +0000 Subject: [PATCH 183/240] Updated from global requirements Change-Id: I9a3940d08ac351e966beb965b0a2ccf3e89c85d5 --- requirements-py2.txt | 2 +- requirements-py3.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-py2.txt b/requirements-py2.txt index e1420074..4da07c81 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -26,5 +26,5 @@ futures>=2.1.6 jsonschema>=2.0.0,<3.0.0 # For common utilities -oslo.utils>=1.1.0 # Apache-2.0 +oslo.utils>=1.2.0 # Apache-2.0 oslo.serialization>=1.0.0 # Apache-2.0 diff --git a/requirements-py3.txt b/requirements-py3.txt index d827a17b..d7f5d0eb 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -20,5 +20,5 @@ stevedore>=1.1.0 # Apache-2.0 jsonschema>=2.0.0,<3.0.0 # For common utilities -oslo.utils>=1.1.0 # Apache-2.0 +oslo.utils>=1.2.0 # Apache-2.0 oslo.serialization>=1.0.0 # Apache-2.0 From a588e484639923cfc339fbe1014bda7043d1bde4 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 9 Jan 2015 12:58:58 -0800 Subject: [PATCH 184/240] Pass a string as executor in the example instead of an executor Instead of creating a throw-away executor and then passing it to the engine for usage there just pass a string that specifies that we want the executor created/used to be process based and let the engine handle the creation and shutdown of that executor on its own. Change-Id: Ib9cdbe50d1a12ec8b5bec86531f76ef51926645c --- taskflow/examples/alphabet_soup.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/taskflow/examples/alphabet_soup.py b/taskflow/examples/alphabet_soup.py index 4e6a0ff9..a287f538 100644 --- a/taskflow/examples/alphabet_soup.py +++ b/taskflow/examples/alphabet_soup.py @@ -22,8 +22,6 @@ import string import sys import time -from concurrent import futures - logging.basicConfig(level=logging.ERROR) self_dir = os.path.abspath(os.path.dirname(__file__)) @@ -46,12 +44,7 @@ from taskflow import task # This is useful since it allows further scaling up your workflows when thread # execution starts to become a bottleneck (which it can start to be due to the # GIL in python). It also offers a intermediary scalable runner that can be -# used when the scale and or setup of remote workers is not desirable. - -# How many local processes to potentially use when executing... (one is fine -# for this example, but more can be used to show play around with what happens -# with many...) -WORKERS = 1 +# used when the scale and/or setup of remote workers is not desirable. def progress_printer(task, event_type, details): @@ -87,15 +80,14 @@ for letter in string.ascii_lowercase: functools.partial(progress_printer, abc)) soup.add(abc) try: - with futures.ProcessPoolExecutor(WORKERS) as executor: - print("Loading...") - e = engines.load(soup, engine='parallel', executor=executor) - print("Compiling...") - e.compile() - print("Preparing...") - e.prepare() - print("Running...") - e.run() - print("Done...") + print("Loading...") + e = engines.load(soup, engine='parallel', executor='processes') + print("Compiling...") + e.compile() + print("Preparing...") + e.prepare() + print("Running...") + e.run() + print("Done...") except exceptions.NotImplementedError as e: print(e) From 96e6d971a02a345d081f98342c47b437db7773e2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 9 Jan 2015 16:35:19 -0800 Subject: [PATCH 185/240] Send in the prior atom state on notification of a state change When a retry atom or a task atom changes state it is quite useful to know the prior state the atom was in; to make this easier pass that information along in the details about the state transition so that observers can inspect it if they desire to. This matches more closely with the flow notification which does include the prior state (under the details key 'old_state') so this also increases the uniformity of notifications in general. Change-Id: I7df1fcc60ba178198776ddaa2681caee5811c4ff --- taskflow/engines/action_engine/actions/retry.py | 11 +++++++---- taskflow/engines/action_engine/actions/task.py | 15 +++++++++------ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/taskflow/engines/action_engine/actions/retry.py b/taskflow/engines/action_engine/actions/retry.py index 5f6eb90c..be933ee2 100644 --- a/taskflow/engines/action_engine/actions/retry.py +++ b/taskflow/engines/action_engine/actions/retry.py @@ -51,21 +51,24 @@ class RetryAction(object): return kwargs def change_state(self, retry, state, result=None): + old_state = self._storage.get_atom_state(retry.name) if state in SAVE_RESULT_STATES: self._storage.save(retry.name, result, state) elif state == states.REVERTED: self._storage.cleanup_retry_history(retry.name, state) else: - old_state = self._storage.get_atom_state(retry.name) if state == old_state: # NOTE(imelnikov): nothing really changed, so we should not # write anything to storage and run notifications return self._storage.set_atom_state(retry.name, state) retry_uuid = self._storage.get_atom_uuid(retry.name) - details = dict(retry_name=retry.name, - retry_uuid=retry_uuid, - result=result) + details = { + 'retry_name': retry.name, + 'retry_uuid': retry_uuid, + 'result': result, + 'old_state': old_state, + } self._notifier.notify(state, details) def execute(self, retry): diff --git a/taskflow/engines/action_engine/actions/task.py b/taskflow/engines/action_engine/actions/task.py index ccf450b5..fbdc0a8f 100644 --- a/taskflow/engines/action_engine/actions/task.py +++ b/taskflow/engines/action_engine/actions/task.py @@ -39,11 +39,10 @@ class TaskAction(object): def handles(atom): return isinstance(atom, task_atom.BaseTask) - def _is_identity_transition(self, state, task, progress): + def _is_identity_transition(self, old_state, state, task, progress): if state in SAVE_RESULT_STATES: # saving result is never identity transition return False - old_state = self._storage.get_atom_state(task.name) if state != old_state: # changing state is not identity transition by definition return False @@ -58,7 +57,8 @@ class TaskAction(object): return True def change_state(self, task, state, result=None, progress=None): - if self._is_identity_transition(state, task, progress): + old_state = self._storage.get_atom_state(task.name) + if self._is_identity_transition(old_state, state, task, progress): # NOTE(imelnikov): ignore identity transitions in order # to avoid extra write to storage backend and, what's # more important, extra notifications @@ -70,9 +70,12 @@ class TaskAction(object): if progress is not None: self._storage.set_task_progress(task.name, progress) task_uuid = self._storage.get_atom_uuid(task.name) - details = dict(task_name=task.name, - task_uuid=task_uuid, - result=result) + details = { + 'task_name': task.name, + 'task_uuid': task_uuid, + 'result': result, + 'old_state': old_state, + } self._notifier.notify(state, details) if progress is not None: task.update_progress(progress) From 0d602a89f3d34055de2e0d39d13ad748c0aa5c2f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 9 Jan 2015 16:58:02 -0800 Subject: [PATCH 186/240] The taskflow logger module does not provide a logging adapter The expected use is that this comparison check for older versions of python (only 2.6) should check against the logging modules logger adapter type and not the taskflow modules helper logger module which does not expose this type. Fixes bug 1409178 Change-Id: I94077bf76d9f13728c6c14b5e263bc7ced3ab0e8 --- taskflow/listeners/logging.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/taskflow/listeners/logging.py b/taskflow/listeners/logging.py index 03055257..c54baf6a 100644 --- a/taskflow/listeners/logging.py +++ b/taskflow/listeners/logging.py @@ -16,6 +16,7 @@ from __future__ import absolute_import +import logging as logging_base import sys from taskflow.listeners import base @@ -36,7 +37,7 @@ else: # when we can just support python 2.7+ (which fixed the lack of this method # on adapters). def _isEnabledFor(logger, level): - if _PY26 and isinstance(logger, logging.LoggerAdapter): + if _PY26 and isinstance(logger, logging_base.LoggerAdapter): return logger.logger.isEnabledFor(level) return logger.isEnabledFor(level) From 2f7d86ac3e606c144f78d1dfa7190cdb0e9c306d Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 10 Jan 2015 14:23:34 -0800 Subject: [PATCH 187/240] Include docstrings for parallel engine types/strings supported When a 'executor' option is passed to a action engine requested to run in parallel mode that option will internally be examined and it will affect the internally used execution model that the engine will use; to make it understandable what the valid options are include a docstring + table(s) that describes the options and there valid values. Change-Id: I9a1852427bae22a01f5993862617e384f10ec005 --- doc/source/engines.rst | 16 +++------ taskflow/engines/action_engine/engine.py | 38 +++++++++++++++++++++- taskflow/engines/action_engine/executor.py | 2 ++ 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/doc/source/engines.rst b/doc/source/engines.rst index 474fa666..04d462a4 100644 --- a/doc/source/engines.rst +++ b/doc/source/engines.rst @@ -161,16 +161,10 @@ Parallel **Engine type**: ``'parallel'`` A parallel engine schedules tasks onto different threads/processes to allow for -running non-dependent tasks simultaneously. - -Additional supported keyword arguments: - -* ``executor``: a object that implements a :pep:`3148` compatible `executor`_ - interface; it will be used for scheduling tasks. You can use instances of a - `thread pool executor`_ or a :py:class:`green executor - ` (which internally uses - `eventlet `_ and greenthread pools) or a - `process pool executor`_. +running non-dependent tasks simultaneously. See the documentation of +:py:class:`~taskflow.engines.action_engine.engine.ParallelActionEngine` for +supported arguments that can be used to construct a parallel engine that runs +using your desired execution model. .. tip:: @@ -340,6 +334,7 @@ Interfaces .. automodule:: taskflow.engines.action_engine.compiler .. automodule:: taskflow.engines.action_engine.completer .. automodule:: taskflow.engines.action_engine.engine +.. automodule:: taskflow.engines.action_engine.executor .. automodule:: taskflow.engines.action_engine.runner .. automodule:: taskflow.engines.action_engine.runtime .. automodule:: taskflow.engines.action_engine.scheduler @@ -360,5 +355,4 @@ Hierarchy .. _executor: https://docs.python.org/dev/library/concurrent.futures.html#concurrent.futures.Executor .. _networkx: https://networkx.github.io/ .. _futures backport: https://pypi.python.org/pypi/futures -.. _thread pool executor: https://docs.python.org/dev/library/concurrent.futures.html#threadpoolexecutor .. _process pool executor: https://docs.python.org/dev/library/concurrent.futures.html#processpoolexecutor diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index 7ececb9f..157f641d 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -239,7 +239,43 @@ class _ExecutorTextMatch(collections.namedtuple('_ExecutorTextMatch', class ParallelActionEngine(ActionEngine): - """Engine that runs tasks in parallel manner.""" + """Engine that runs tasks in parallel manner. + + Supported keyword arguments: + + * ``executor``: a object that implements a :pep:`3148` compatible executor + interface; it will be used for scheduling tasks. The following + type are applicable (other unknown types passed will cause a type + error to be raised). + +========================= =============================================== +Type provided Executor used +========================= =============================================== +|cft|.ThreadPoolExecutor :class:`~.executor.ParallelThreadTaskExecutor` +|cfp|.ProcessPoolExecutor :class:`~.executor.ParallelProcessTaskExecutor` +|cf|._base.Executor :class:`~.executor.ParallelThreadTaskExecutor` +========================= =============================================== + + * ``executor``: a string that will be used to select a :pep:`3148` + compatible executor; it will be used for scheduling tasks. The following + string are applicable (other unknown strings passed will cause a value + error to be raised). + +=========================== =============================================== +String (case insensitive) Executor used +=========================== =============================================== +``process`` :class:`~.executor.ParallelProcessTaskExecutor` +``processes`` :class:`~.executor.ParallelProcessTaskExecutor` +``thread`` :class:`~.executor.ParallelThreadTaskExecutor` +``threaded`` :class:`~.executor.ParallelThreadTaskExecutor` +``threads`` :class:`~.executor.ParallelThreadTaskExecutor` +=========================== =============================================== + + .. |cfp| replace:: concurrent.futures.process + .. |cft| replace:: concurrent.futures.thread + .. |cf| replace:: concurrent.futures + """ + _storage_factory = atom_storage.MultiThreadedStorage # One of these types should match when a object (non-string) is provided diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index 8a31fddf..773c67b3 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -373,6 +373,7 @@ class ParallelTaskExecutor(TaskExecutor): to concurrent.Futures.Executor. """ + #: Options this executor supports (passed in from engine options). OPTIONS = frozenset(['max_workers']) def __init__(self, executor=None, max_workers=None): @@ -431,6 +432,7 @@ class ParallelProcessTaskExecutor(ParallelTaskExecutor): the parent are executed on events in the child. """ + #: Options this executor supports (passed in from engine options). OPTIONS = frozenset(['max_workers', 'dispatch_periodicity']) def __init__(self, executor=None, max_workers=None, From c07a96b3b8c0726db894365bcbf0cd1a8b51ddce Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 10 Jan 2015 15:40:30 -0800 Subject: [PATCH 188/240] Update the README.rst with accurate requirements We no longer have a 'optional-requirements.txt' file so the README.rst should not mention it existing (since it's not there anymore) so this updates that file with valid requirements and where they are located (and includes examples of optional requirements with links to there project pages). Change-Id: Idb5fa3db8fc55027993b737cf251bd5257ab87be --- README.rst | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index a19bc4d2..c5e1f105 100644 --- a/README.rst +++ b/README.rst @@ -21,16 +21,19 @@ Testing and requirements Requirements ~~~~~~~~~~~~ -Because TaskFlow has many optional (pluggable) parts like persistence -backends and engines, we decided to split our requirements into two -parts: - things that are absolutely required by TaskFlow (you can't use -TaskFlow without them) are put into ``requirements-pyN.txt`` (``N`` being the -Python *major* version number used to install the package); - things that are -required by some optional part of TaskFlow (you can use TaskFlow without -them) are put into ``optional-requirements.txt``; if you want to use the -feature in question, you should add that requirements to your project or -environment; - as usual, things that required only for running tests are -put into ``test-requirements.txt``. +Because this project has many optional (pluggable) parts like persistence +backends and engines, we decided to split our requirements into three +parts: - things that are absolutely required (you can't use the project +without them) are put into ``requirements-pyN.txt`` (``N`` being the +Python *major* version number used to install the package). The requirements +that are required by some optional part of this project (you can use the +project without them) are put into our ``tox.ini`` file (so that we can still +test the optional functionality works as expected). If you want to use the +feature in question (`eventlet`_ or the worker based engine that +uses `kombu`_ or the `sqlalchemy`_ persistence backend or jobboards which +have an implementation built using `kazoo`_ ...), you should add +that requirement(s) to your project or environment; - as usual, things that +required only for running tests are put into ``test-requirements.txt``. Tox.ini ~~~~~~~ @@ -51,5 +54,9 @@ We also have sphinx documentation in ``docs/source``. $ python setup.py build_sphinx -.. _tox: http://testrun.org/tox/latest/ +.. _kazoo: http://kazoo.readthedocs.org/ +.. _sqlalchemy: http://www.sqlalchemy.org/ +.. _kombu: http://kombu.readthedocs.org/ +.. _eventlet: http://eventlet.net/ +.. _tox: http://tox.testrun.org/ .. _developer documentation: http://docs.openstack.org/developer/taskflow/ From 778e210e17e2a3858f644e7f11418f6ad2c465f1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 9 Jan 2015 16:48:37 -0800 Subject: [PATCH 189/240] Include the 'old_state' in all currently provided listeners Since the key 'old_state' is now standard across all notifications sent from flows, retries or tasks we can safely use it in the listeners without worrying about hitting key errors when it does not exist/is not provided. This also adds an example which shows how to use the dynamic logging listener to view this information. Change-Id: If4456440674347a3fe4972d9d0fa16ba8373ef9f --- doc/source/examples.rst | 12 +++++++ doc/source/notifications.rst | 12 +++---- taskflow/examples/echo_listener.py | 56 ++++++++++++++++++++++++++++++ taskflow/listeners/base.py | 18 +++++----- taskflow/listeners/logging.py | 21 ++++++----- 5 files changed, 95 insertions(+), 24 deletions(-) create mode 100644 taskflow/examples/echo_listener.py diff --git a/doc/source/examples.rst b/doc/source/examples.rst index f839e353..d30bd85f 100644 --- a/doc/source/examples.rst +++ b/doc/source/examples.rst @@ -22,6 +22,18 @@ Passing values from and to tasks :linenos: :lines: 16- +Using listeners +=============== + +.. note:: + + Full source located at :example:`echo_listener`. + +.. literalinclude:: ../../taskflow/examples/echo_listener.py + :language: python + :linenos: + :lines: 16- + Making phone calls ================== diff --git a/doc/source/notifications.rst b/doc/source/notifications.rst index 13c550ee..3fd35e2b 100644 --- a/doc/source/notifications.rst +++ b/doc/source/notifications.rst @@ -136,14 +136,14 @@ For example, this is how you can use >>> with printing.PrintingListener(eng): ... eng.run() ... - has moved flow 'cat-dog' (...) into state 'RUNNING' - has moved task 'CatTalk' (...) into state 'RUNNING' + has moved flow 'cat-dog' (...) into state 'RUNNING' from state 'PENDING' + has moved task 'CatTalk' (...) into state 'RUNNING' from state 'PENDING' meow - has moved task 'CatTalk' (...) into state 'SUCCESS' with result 'cat' (failure=False) - has moved task 'DogTalk' (...) into state 'RUNNING' + has moved task 'CatTalk' (...) into state 'SUCCESS' from state 'RUNNING' with result 'cat' (failure=False) + has moved task 'DogTalk' (...) into state 'RUNNING' from state 'PENDING' woof - has moved task 'DogTalk' (...) into state 'SUCCESS' with result 'dog' (failure=False) - has moved flow 'cat-dog' (...) into state 'SUCCESS' + has moved task 'DogTalk' (...) into state 'SUCCESS' from state 'RUNNING' with result 'dog' (failure=False) + has moved flow 'cat-dog' (...) into state 'SUCCESS' from state 'RUNNING' Basic listener -------------- diff --git a/taskflow/examples/echo_listener.py b/taskflow/examples/echo_listener.py new file mode 100644 index 00000000..a8eebf60 --- /dev/null +++ b/taskflow/examples/echo_listener.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import sys + +logging.basicConfig(level=logging.DEBUG) + +top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, + os.pardir)) +sys.path.insert(0, top_dir) + +from taskflow import engines +from taskflow.listeners import logging as logging_listener +from taskflow.patterns import linear_flow as lf +from taskflow import task + +# INTRO: This example walks through a miniature workflow which will do a +# simple echo operation; during this execution a listener is assocated with +# the engine to recieve all notifications about what the flow has performed, +# this example dumps that output to the stdout for viewing (at debug level +# to show all the information which is possible). + + +class Echo(task.Task): + def execute(self): + print(self.name) + + +# Generate the work to be done (but don't do it yet). +wf = lf.Flow('abc') +wf.add(Echo('a')) +wf.add(Echo('b')) +wf.add(Echo('c')) + +# This will associate the listener with the engine (the listener +# will automatically register for notifications with the engine and deregister +# when the context is exited). +e = engines.load(wf) +with logging_listener.DynamicLoggingListener(e): + e.run() diff --git a/taskflow/listeners/base.py b/taskflow/listeners/base.py index 0101e8de..42dc87e9 100644 --- a/taskflow/listeners/base.py +++ b/taskflow/listeners/base.py @@ -187,9 +187,9 @@ class DumpingListener(Listener): """Dumps the provided *templated* message to some output.""" def _flow_receiver(self, state, details): - self._dump("%s has moved flow '%s' (%s) into state '%s'", - self._engine, details['flow_name'], - details['flow_uuid'], state) + self._dump("%s has moved flow '%s' (%s) into state '%s'" + " from state '%s'", self._engine, details['flow_name'], + details['flow_uuid'], state, details['old_state']) def _task_receiver(self, state, details): if state in FINISH_STATES: @@ -201,14 +201,14 @@ class DumpingListener(Listener): exc_info = tuple(result.exc_info) was_failure = True self._dump("%s has moved task '%s' (%s) into state '%s'" - " with result '%s' (failure=%s)", + " from state '%s' with result '%s' (failure=%s)", self._engine, details['task_name'], - details['task_uuid'], state, result, was_failure, - exc_info=exc_info) + details['task_uuid'], state, details['old_state'], + result, was_failure, exc_info=exc_info) else: - self._dump("%s has moved task '%s' (%s) into state '%s'", - self._engine, details['task_name'], - details['task_uuid'], state) + self._dump("%s has moved task '%s' (%s) into state '%s'" + " from state '%s'", self._engine, details['task_name'], + details['task_uuid'], state, details['old_state']) # TODO(harlowja): remove in 0.7 or later... diff --git a/taskflow/listeners/logging.py b/taskflow/listeners/logging.py index 03055257..3245d2ba 100644 --- a/taskflow/listeners/logging.py +++ b/taskflow/listeners/logging.py @@ -167,8 +167,9 @@ class DynamicLoggingListener(base.Listener): exc_info, exc_details = self._format_failure(result) self._logger.log(self._failure_level, "Task '%s' (%s) transitioned into state" - " '%s'%s", details['task_name'], - details['task_uuid'], state, exc_details, + " '%s' from state '%s'%s", + details['task_name'], details['task_uuid'], + state, details['old_state'], exc_details, exc_info=exc_info) else: # Otherwise, depending on the enabled logging level/state we @@ -178,17 +179,19 @@ class DynamicLoggingListener(base.Listener): if (_isEnabledFor(self._logger, self._level) or state == states.FAILURE): self._logger.log(level, "Task '%s' (%s) transitioned into" - " state '%s' with result '%s'", - details['task_name'], + " state '%s' from state '%s' with" + " result '%s'", details['task_name'], details['task_uuid'], state, - result) + details['old_state'], result) else: self._logger.log(level, "Task '%s' (%s) transitioned into" - " state '%s'", details['task_name'], - details['task_uuid'], state) + " state '%s' from state '%s'", + details['task_name'], + details['task_uuid'], state, + details['old_state']) else: # Just a intermediary state, carry on! level = self._task_log_levels.get(state, self._level) self._logger.log(level, "Task '%s' (%s) transitioned into state" - " '%s'", details['task_name'], - details['task_uuid'], state) + " '%s' from state '%s'", details['task_name'], + details['task_uuid'], state, details['old_state']) From 45c28bd482971a6cf324ff987114b380e581d1c7 Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Tue, 13 Jan 2015 00:16:34 +0000 Subject: [PATCH 190/240] Updated from global requirements Change-Id: I2a471d6d1c70d77784c0370ccb3de547590031cf --- requirements-py2.txt | 2 +- requirements-py3.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-py2.txt b/requirements-py2.txt index 4da07c81..083caec0 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -27,4 +27,4 @@ jsonschema>=2.0.0,<3.0.0 # For common utilities oslo.utils>=1.2.0 # Apache-2.0 -oslo.serialization>=1.0.0 # Apache-2.0 +oslo.serialization>=1.2.0 # Apache-2.0 diff --git a/requirements-py3.txt b/requirements-py3.txt index d7f5d0eb..b04fc0af 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -21,4 +21,4 @@ jsonschema>=2.0.0,<3.0.0 # For common utilities oslo.utils>=1.2.0 # Apache-2.0 -oslo.serialization>=1.0.0 # Apache-2.0 +oslo.serialization>=1.2.0 # Apache-2.0 From 4c756ef852d62239f07d8dfbf6773512e92053c0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 12 Jan 2015 18:13:59 -0800 Subject: [PATCH 191/240] Use a single sender Instead of register many different senders (one for each needed proxy notification type) just use a single sender for all the different types to save space and time when the senders are pickled across to the worker(s). Change-Id: I1e1ab6c708c855e5868061bc338d399a38ad954e --- taskflow/engines/action_engine/executor.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index 8a31fddf..dafc4f58 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -487,8 +487,10 @@ class ParallelProcessTaskExecutor(ParallelTaskExecutor): needed.add(event_type) if progress_callback is not None: needed.add(_UPDATE_PROGRESS) - for event_type in needed: - clone.notifier.register(event_type, _EventSender(channel)) + if needed: + sender = _EventSender(channel) + for event_type in needed: + clone.notifier.register(event_type, sender) def _submit_task(self, func, task, *args, **kwargs): """Submit a function to run the given task (with given args/kwargs). From eb536daa0e63aa2e110ac48ff5da65b27b48473c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 12 Jan 2015 18:25:14 -0800 Subject: [PATCH 192/240] Create and use a multiprocessing sync manager subclass Instead of accessing private variables of the manager base class just create a subclass and more nicely expose methods that can be used to introspect the managers state and perform actions based on that state. Change-Id: Ied570a25e52de94370b59d844ecdcc6d3551fd3d --- taskflow/engines/action_engine/executor.py | 23 ++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index dafc4f58..3af95999 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -16,7 +16,6 @@ import abc import collections -import multiprocessing from multiprocessing import managers import os import pickle @@ -94,6 +93,16 @@ def _revert_task(task, arguments, result, failures, progress_callback=None): return (REVERTED, result) +class _ViewableSyncManager(managers.SyncManager): + """Manager that exposes its state as methods.""" + + def is_shutdown(self): + return self._state.value == managers.State.SHUTDOWN + + def is_running(self): + return self._state.value == managers.State.STARTED + + class _Channel(object): """Helper wrapper around a multiprocessing queue used by a worker.""" @@ -437,7 +446,7 @@ class ParallelProcessTaskExecutor(ParallelTaskExecutor): dispatch_periodicity=None): super(ParallelProcessTaskExecutor, self).__init__( executor=executor, max_workers=max_workers) - self._manager = multiprocessing.Manager() + self._manager = _ViewableSyncManager() self._dispatcher = _Dispatcher( dispatch_periodicity=dispatch_periodicity) # Only created after starting... @@ -452,12 +461,10 @@ class ParallelProcessTaskExecutor(ParallelTaskExecutor): raise RuntimeError("Worker thread must be stopped via stop()" " before starting/restarting") super(ParallelProcessTaskExecutor, self).start() - # TODO(harlowja): do something else here besides accessing a state - # of the manager internals (it doesn't seem to expose any way to know - # this information)... - if self._manager._state.value == managers.State.SHUTDOWN: - self._manager = multiprocessing.Manager() - if self._manager._state.value == managers.State.INITIAL: + # These don't seem restartable; make a new one... + if self._manager.is_shutdown(): + self._manager = _ViewableSyncManager() + if not self._manager.is_running(): self._manager.start() self._dispatcher.reset() self._queue = self._manager.Queue() From 42a665d06f5a75672417a28375ca4e1988365aa8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 11 Jan 2015 13:03:26 -0800 Subject: [PATCH 193/240] Use platform neutral line separator(s) To at least try to support things like windows it's better if we can make an attempt to use the platform neutral characters for line separator(s) where appropriate. Change-Id: Icc533ed4d4c94f461b7f19600b74146221f17b18 --- taskflow/engines/worker_based/worker.py | 9 +++++++-- taskflow/exceptions.py | 10 ++++++++-- taskflow/listeners/claims.py | 4 +++- taskflow/listeners/logging.py | 3 ++- taskflow/types/failure.py | 8 ++++++-- taskflow/types/graph.py | 3 ++- taskflow/types/table.py | 12 +++++++----- taskflow/types/tree.py | 4 +++- taskflow/utils/misc.py | 5 +++++ taskflow/utils/persistence_utils.py | 7 ++++--- 10 files changed, 47 insertions(+), 18 deletions(-) diff --git a/taskflow/engines/worker_based/worker.py b/taskflow/engines/worker_based/worker.py index f75b7a8d..5464c814 100644 --- a/taskflow/engines/worker_based/worker.py +++ b/taskflow/engines/worker_based/worker.py @@ -141,8 +141,13 @@ class Worker(object): pass tpl_params['platform'] = platform.platform() tpl_params['thread_id'] = tu.get_ident() - return BANNER_TEMPLATE.substitute(BANNER_TEMPLATE.defaults, - **tpl_params) + banner = BANNER_TEMPLATE.substitute(BANNER_TEMPLATE.defaults, + **tpl_params) + # NOTE(harlowja): this is needed since the template in this file + # will always have newlines that end with '\n' (even on different + # platforms due to the way this source file is encoded) so we have + # to do this little dance to make it platform neutral... + return misc.fix_newlines(banner) def run(self, display_banner=True, banner_writer=None): """Runs the worker.""" diff --git a/taskflow/exceptions.py b/taskflow/exceptions.py index c6632754..d4db4e5f 100644 --- a/taskflow/exceptions.py +++ b/taskflow/exceptions.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +import os import traceback import six @@ -46,14 +47,19 @@ class TaskFlowException(Exception): """Pretty formats a taskflow exception + any connected causes.""" if indent < 0: raise ValueError("indent must be greater than or equal to zero") - return "\n".join(self._pformat(self, [], 0, - indent=indent, indent_text=indent_text)) + return os.linesep.join(self._pformat(self, [], 0, + indent=indent, + indent_text=indent_text)) @classmethod def _pformat(cls, excp, lines, current_indent, indent=2, indent_text=" "): line_prefix = indent_text * current_indent for line in traceback.format_exception_only(type(excp), excp): # We'll add our own newlines on at the end of formatting. + # + # NOTE(harlowja): the reason we don't search for os.linesep is + # that the traceback module seems to only use '\n' (for some + # reason). if line.endswith("\n"): line = line[0:-1] lines.append(line_prefix + line) diff --git a/taskflow/listeners/claims.py b/taskflow/listeners/claims.py index 7fa86474..82b89cf0 100644 --- a/taskflow/listeners/claims.py +++ b/taskflow/listeners/claims.py @@ -17,6 +17,7 @@ from __future__ import absolute_import import logging +import os import six @@ -70,7 +71,8 @@ class CheckingClaimListener(base.Listener): engine.suspend() except exceptions.TaskFlowException as e: LOG.warn("Failed suspending engine '%s', (previously owned by" - " '%s'):\n%s", engine, self._owner, e.pformat()) + " '%s'):%s%s", engine, self._owner, os.linesep, + e.pformat()) def _flow_receiver(self, state, details): self._claim_checker(state, details) diff --git a/taskflow/listeners/logging.py b/taskflow/listeners/logging.py index c54baf6a..4995e9f2 100644 --- a/taskflow/listeners/logging.py +++ b/taskflow/listeners/logging.py @@ -17,6 +17,7 @@ from __future__ import absolute_import import logging as logging_base +import os import sys from taskflow.listeners import base @@ -148,7 +149,7 @@ class DynamicLoggingListener(base.Listener): # exc_info that can be used but we *should* have a string # version that we can use instead... exc_info = None - exc_details = "\n%s" % fail.pformat(traceback=True) + exc_details = "%s%s" % (os.linesep, fail.pformat(traceback=True)) return (exc_info, exc_details) def _flow_receiver(self, state, details): diff --git a/taskflow/types/failure.py b/taskflow/types/failure.py index aba04d4a..ae848d31 100644 --- a/taskflow/types/failure.py +++ b/taskflow/types/failure.py @@ -15,6 +15,7 @@ # under the License. import copy +import os import sys import traceback @@ -280,10 +281,13 @@ class Failure(object): else: traceback_str = None if traceback_str: - buf.write('\nTraceback (most recent call last):\n') + buf.write(os.linesep) + buf.write('Traceback (most recent call last):') + buf.write(os.linesep) buf.write(traceback_str) else: - buf.write('\nTraceback not available.') + buf.write(os.linesep) + buf.write('Traceback not available.') return buf.getvalue() def __iter__(self): diff --git a/taskflow/types/graph.py b/taskflow/types/graph.py index 22b6b71c..068a8e20 100644 --- a/taskflow/types/graph.py +++ b/taskflow/types/graph.py @@ -15,6 +15,7 @@ # under the License. import collections +import os import networkx as nx import six @@ -78,7 +79,7 @@ class DiGraph(nx.DiGraph): buf.write(" --> %s" % (cycle[i])) buf.write(" --> %s" % (cycle[0])) lines.append(" %s" % buf.getvalue()) - return "\n".join(lines) + return os.linesep.join(lines) def export_to_dot(self): """Exports the graph to a dot format (requires pydot library).""" diff --git a/taskflow/types/table.py b/taskflow/types/table.py index b6257200..6813fab1 100644 --- a/taskflow/types/table.py +++ b/taskflow/types/table.py @@ -15,6 +15,7 @@ # under the License. import itertools +import os import six @@ -39,6 +40,7 @@ class PleasantTable(object): COLUMN_SEPARATOR_CHAR = '|' HEADER_FOOTER_JOINING_CHAR = '+' HEADER_FOOTER_CHAR = '-' + LINE_SEP = os.linesep @staticmethod def _center_text(text, max_len, fill=' '): @@ -87,7 +89,7 @@ class PleasantTable(object): # Build the main header. content_buf = six.StringIO() content_buf.write(header_footer_buf.getvalue()) - content_buf.write("\n") + content_buf.write(self.LINE_SEP) content_buf.write(self.COLUMN_STARTING_CHAR) for i, header in enumerate(headers): if i + 1 == column_count: @@ -99,12 +101,12 @@ class PleasantTable(object): else: content_buf.write(headers[i]) content_buf.write(self.COLUMN_SEPARATOR_CHAR) - content_buf.write("\n") + content_buf.write(self.LINE_SEP) content_buf.write(header_footer_buf.getvalue()) # Build the main content. row_count = len(self._rows) if row_count: - content_buf.write("\n") + content_buf.write(self.LINE_SEP) for i, row in enumerate(self._rows): pieces = [] for j, column in enumerate(row): @@ -122,7 +124,7 @@ class PleasantTable(object): content_buf.write(self.COLUMN_STARTING_CHAR) content_buf.write(blob) if i + 1 != row_count: - content_buf.write("\n") - content_buf.write("\n") + content_buf.write(self.LINE_SEP) + content_buf.write(self.LINE_SEP) content_buf.write(header_footer_buf.getvalue()) return content_buf.getvalue() diff --git a/taskflow/types/tree.py b/taskflow/types/tree.py index 6eb960b4..e6fad20c 100644 --- a/taskflow/types/tree.py +++ b/taskflow/types/tree.py @@ -16,6 +16,8 @@ # License for the specific language governing permissions and limitations # under the License. +import os + import six @@ -148,7 +150,7 @@ class Node(object): for i, line in enumerate(_inner_pformat(self, 0)): accumulator.write(line) if i < expected_lines: - accumulator.write('\n') + accumulator.write(os.linesep) return accumulator.getvalue() def child_count(self, only_direct=True): diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index dd3610e7..5aa62fd9 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -142,6 +142,11 @@ def clamp(value, minimum, maximum, on_clamped=None): return value +def fix_newlines(text, replacement=os.linesep): + """Fixes text that *may* end with wrong nl by replacing with right nl.""" + return replacement.join(text.splitlines()) + + def binary_encode(text, encoding='utf-8'): """Converts a string of into a binary type using given encoding. diff --git a/taskflow/utils/persistence_utils.py b/taskflow/utils/persistence_utils.py index b8a15351..6d5d1685 100644 --- a/taskflow/utils/persistence_utils.py +++ b/taskflow/utils/persistence_utils.py @@ -15,6 +15,7 @@ # under the License. import contextlib +import os from oslo.utils import timeutils from oslo.utils import uuidutils @@ -139,7 +140,7 @@ def pformat_atom_detail(atom_detail, indent=0): lines.append("%s- failure = %s" % (" " * (indent + 1), bool(atom_detail.failure))) lines.extend(_format_meta(atom_detail.meta, indent=indent + 1)) - return "\n".join(lines) + return os.linesep.join(lines) def pformat_flow_detail(flow_detail, indent=0): @@ -149,7 +150,7 @@ def pformat_flow_detail(flow_detail, indent=0): lines.extend(_format_meta(flow_detail.meta, indent=indent + 1)) for task_detail in flow_detail: lines.append(pformat_atom_detail(task_detail, indent=indent + 1)) - return "\n".join(lines) + return os.linesep.join(lines) def pformat(book, indent=0): @@ -167,4 +168,4 @@ def pformat(book, indent=0): timeutils.isotime(book.updated_at))) for flow_detail in book: lines.append(pformat_flow_detail(flow_detail, indent=indent + 1)) - return "\n".join(lines) + return os.linesep.join(lines) From 9fe99ba7106c4de5655c40c28d175e136ddd29d2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 12 Jan 2015 19:57:50 -0800 Subject: [PATCH 194/240] Add split time capturing to the stop watch For cases where it is useful to capture the elapsed time of a watch and later examine those split times create a method on the stop watch that allows for these kinds of captures and iterations that correspond to the common stop watch split capability. Also adds basic docstrings to the stop watch public methods so that people know what they are and can use them. Change-Id: Ic52ae5dfcca9a117ccd0dda5cc62a14c09e15ce0 --- taskflow/tests/unit/test_types.py | 24 ++++++++++++ taskflow/types/timing.py | 61 +++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/taskflow/tests/unit/test_types.py b/taskflow/tests/unit/test_types.py index 28b57251..85d6e557 100644 --- a/taskflow/tests/unit/test_types.py +++ b/taskflow/tests/unit/test_types.py @@ -193,6 +193,30 @@ class StopWatchTest(test.TestCase): timeutils.advance_time_seconds(0.05) self.assertGreater(0.01, watch.elapsed()) + def test_splits(self): + watch = tt.StopWatch() + watch.start() + self.assertEqual(0, len(watch.splits)) + + watch.split() + self.assertEqual(1, len(watch.splits)) + self.assertEqual(watch.splits[0].elapsed, + watch.splits[0].length) + + timeutils.advance_time_seconds(0.05) + watch.split() + splits = watch.splits + self.assertEqual(2, len(splits)) + self.assertNotEqual(splits[0].elapsed, splits[1].elapsed) + self.assertEqual(splits[1].length, + splits[1].elapsed - splits[0].elapsed) + + watch.stop() + self.assertEqual(2, len(watch.splits)) + + watch.start() + self.assertEqual(0, len(watch.splits)) + class TableTest(test.TestCase): def test_create_valid_no_rows(self): diff --git a/taskflow/types/timing.py b/taskflow/types/timing.py index c7cb38db..2cbeee9b 100644 --- a/taskflow/types/timing.py +++ b/taskflow/types/timing.py @@ -44,6 +44,34 @@ class Timeout(object): self._event.clear() +class Split(object): + """A *immutable* stopwatch split. + + See: http://en.wikipedia.org/wiki/Stopwatch for what this is/represents. + """ + + __slots__ = ['_elapsed', '_length'] + + def __init__(self, elapsed, length): + self._elapsed = elapsed + self._length = length + + @property + def elapsed(self): + """Duration from stopwatch start.""" + return self._elapsed + + @property + def length(self): + """Seconds from last split (or the elapsed time if no prior split).""" + return self._length + + def __repr__(self): + r = self.__class__.__name__ + r += "(elapsed=%s, length=%s)" % (self._elapsed, self._length) + return r + + class StopWatch(object): """A simple timer/stopwatch helper class. @@ -67,22 +95,49 @@ class StopWatch(object): self._started_at = None self._stopped_at = None self._state = None + self._splits = [] def start(self): + """Starts the watch (if not already started). + + NOTE(harlowja): resets any splits previously captured (if any). + """ if self._state == self._STARTED: return self self._started_at = timeutils.utcnow() self._stopped_at = None self._state = self._STARTED + self._splits = [] return self + @property + def splits(self): + """Accessor to all/any splits that have been captured.""" + return tuple(self._splits) + + def split(self): + """Captures a split/elapsed since start time (and doesn't stop).""" + if self._state == self._STARTED: + elapsed = self.elapsed() + if self._splits: + length = max(0.0, elapsed - self._splits[-1].elapsed) + else: + length = elapsed + self._splits.append(Split(elapsed, length)) + return self._splits[-1] + else: + raise RuntimeError("Can not create a split time of a stopwatch" + " if it has not been started") + def restart(self): + """Restarts the watch from a started/stopped state.""" if self._state == self._STARTED: self.stop() self.start() return self def elapsed(self): + """Returns how many seconds have elapsed.""" if self._state == self._STOPPED: return max(0.0, float(timeutils.delta_seconds(self._started_at, self._stopped_at))) @@ -94,16 +149,19 @@ class StopWatch(object): " if it has not been started/stopped") def __enter__(self): + """Starts the watch.""" self.start() return self def __exit__(self, type, value, traceback): + """Stops the watch (ignoring errors if stop fails).""" try: self.stop() except RuntimeError: pass def leftover(self): + """Returns how many seconds are left until the watch expires.""" if self._duration is None: raise RuntimeError("Can not get the leftover time of a watch that" " has no duration") @@ -113,6 +171,7 @@ class StopWatch(object): return max(0.0, self._duration - self.elapsed()) def expired(self): + """Returns if the watch has expired (ie, duration provided elapsed).""" if self._duration is None: return False if self._state is None: @@ -123,6 +182,7 @@ class StopWatch(object): return False def resume(self): + """Resumes the watch from a stopped state.""" if self._state == self._STOPPED: self._state = self._STARTED return self @@ -131,6 +191,7 @@ class StopWatch(object): " stopped") def stop(self): + """Stops the watch.""" if self._state == self._STOPPED: return self if self._state != self._STARTED: From bf2f205e163cb08e2717f4fd5f2997215088126d Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 13 Jan 2015 11:24:00 -0800 Subject: [PATCH 195/240] Use oslo.utils reflection for class name Use the utility function provided by oslo.utils instead of trying to get the class name via the self class name variable. Change-Id: Ib113d557f08797010a774c36e0ad4a6e14e7e058 --- taskflow/types/timing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/taskflow/types/timing.py b/taskflow/types/timing.py index 2cbeee9b..ab1d39dd 100644 --- a/taskflow/types/timing.py +++ b/taskflow/types/timing.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.utils import reflection from oslo.utils import timeutils from taskflow.utils import threading_utils @@ -67,7 +68,7 @@ class Split(object): return self._length def __repr__(self): - r = self.__class__.__name__ + r = reflection.get_class_name(self, fully_qualified=False) r += "(elapsed=%s, length=%s)" % (self._elapsed, self._length) return r From 9c15efff8ce509fe81f4e33a3c9e2c51cd91432c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 11 Jan 2015 22:12:55 -0800 Subject: [PATCH 196/240] Add executor statistics It is quite often useful to know various data about things submitted to an executor, so to enable this data we first have to gather it. This adds on that needed collection to our executor subtypes so that we gather the following: - How many submissions failed (with exceptions) - How many submissions were cancelled before executing - How many submisions were executed (failed or not) - How long submissions that were executed ran for Change-Id: I9e3a77296201c4c66d439891e3f5b71f834d441a --- taskflow/tests/unit/test_futures.py | 11 +- taskflow/types/futures.py | 176 +++++++++++++++++++++++++++- 2 files changed, 180 insertions(+), 7 deletions(-) diff --git a/taskflow/tests/unit/test_futures.py b/taskflow/tests/unit/test_futures.py index 576b5eee..437bf28a 100644 --- a/taskflow/tests/unit/test_futures.py +++ b/taskflow/tests/unit/test_futures.py @@ -67,6 +67,7 @@ class _SimpleFuturesTestMixin(object): with self._make_executor(2) as e: f = e.submit(_blowup) self.assertRaises(IOError, f.result) + self.assertEqual(1, e.statistics.failures) def test_accumulator(self): created = [] @@ -75,6 +76,7 @@ class _SimpleFuturesTestMixin(object): created.append(e.submit(_return_one)) results = [f.result() for f in created] self.assertEqual(10, sum(results)) + self.assertEqual(10, e.statistics.executed) def test_map(self): count = [i for i in range(0, 100)] @@ -119,6 +121,7 @@ class _FuturesTestMixin(_SimpleFuturesTestMixin): self.assertEqual(1, called[0]) self.assertEqual(1, called[1]) + self.assertEqual(2, e.statistics.executed) def test_result_callback(self): called = collections.defaultdict(int) @@ -143,19 +146,22 @@ class _FuturesTestMixin(_SimpleFuturesTestMixin): for i in range(0, create_am): fs.append(e.submit(functools.partial(_return_given, i))) self.assertEqual(create_am, len(fs)) + self.assertEqual(create_am, e.statistics.executed) for i in range(0, create_am): result = fs[i].result() self.assertEqual(i, result) def test_called_restricted_size(self): + create_am = 100 called = collections.defaultdict(int) with self._make_executor(1) as e: - for f in self._make_funcs(called, 100): + for f in self._make_funcs(called, create_am): e.submit(f) self.assertFalse(e.alive) - self.assertEqual(100, len(called)) + self.assertEqual(create_am, len(called)) + self.assertEqual(create_am, e.statistics.executed) class ThreadPoolExecutorTest(test.TestCase, _FuturesTestMixin): @@ -217,6 +223,7 @@ class GreenThreadPoolExecutorTest(test.TestCase, _FuturesTestMixin): self.assertEqual(0, len(called)) self.assertEqual(2, len(fs)) + self.assertEqual(2, e.statistics.cancelled) for f in fs: self.assertTrue(f.cancelled()) self.assertTrue(f.done()) diff --git a/taskflow/types/futures.py b/taskflow/types/futures.py index 194730e5..8e8e67af 100644 --- a/taskflow/types/futures.py +++ b/taskflow/types/futures.py @@ -14,9 +14,13 @@ # License for the specific language governing permissions and limitations # under the License. +import functools +import threading + from concurrent import futures as _futures from concurrent.futures import process as _process from concurrent.futures import thread as _thread +from oslo.utils import reflection try: from eventlet.green import threading as greenthreading @@ -27,6 +31,7 @@ try: except ImportError: EVENTLET_AVAILABLE = False +from taskflow.types import timing from taskflow.utils import threading_utils as tu @@ -34,9 +39,59 @@ from taskflow.utils import threading_utils as tu Future = _futures.Future +class _Gatherer(object): + def __init__(self, submit_func, + lock_cls=threading.Lock, start_before_submit=False): + self._submit_func = submit_func + self._stats_lock = lock_cls() + self._stats = ExecutorStatistics() + self._start_before_submit = start_before_submit + + @property + def statistics(self): + return self._stats + + def _capture_stats(self, watch, fut): + watch.stop() + with self._stats_lock: + # Use a new collection and lock so that all mutations are seen as + # atomic and not overlapping and corrupting with other + # mutations (the clone ensures that others reading the current + # values will not see a mutated/corrupted one). Since futures may + # be completed by different threads we need to be extra careful to + # gather this data in a way that is thread-safe... + (failures, executed, runtime, cancelled) = (self._stats.failures, + self._stats.executed, + self._stats.runtime, + self._stats.cancelled) + if fut.cancelled(): + cancelled += 1 + else: + executed += 1 + if fut.exception() is not None: + failures += 1 + runtime += watch.elapsed() + self._stats = ExecutorStatistics(failures=failures, + executed=executed, + runtime=runtime, + cancelled=cancelled) + + def submit(self, fn, *args, **kwargs): + watch = timing.StopWatch() + if self._start_before_submit: + watch.start() + fut = self._submit_func(fn, *args, **kwargs) + if not self._start_before_submit: + watch.start() + fut.add_done_callback(functools.partial(self._capture_stats, watch)) + return fut + + class ThreadPoolExecutor(_thread.ThreadPoolExecutor): """Executor that uses a thread pool to execute calls asynchronously. + It gathers statistics about the submissions executed for post-analysis... + See: https://docs.python.org/dev/library/concurrent.futures.html """ def __init__(self, max_workers=None): @@ -45,15 +100,31 @@ class ThreadPoolExecutor(_thread.ThreadPoolExecutor): super(ThreadPoolExecutor, self).__init__(max_workers=max_workers) if self._max_workers <= 0: raise ValueError("Max workers must be greater than zero") + self._gatherer = _Gatherer( + # Since our submit will use this gatherer we have to reference + # the parent submit, bound to this instance (which is what we + # really want to use anyway). + super(ThreadPoolExecutor, self).submit) + + @property + def statistics(self): + """:class:`.ExecutorStatistics` about the executors executions.""" + return self._gatherer.statistics @property def alive(self): return not self._shutdown + def submit(self, fn, *args, **kwargs): + """Submit some work to be executed (and gather statistics).""" + return self._gatherer.submit(fn, *args, **kwargs) + class ProcessPoolExecutor(_process.ProcessPoolExecutor): """Executor that uses a process pool to execute calls asynchronously. + It gathers statistics about the submissions executed for post-analysis... + See: https://docs.python.org/dev/library/concurrent.futures.html """ def __init__(self, max_workers=None): @@ -62,11 +133,25 @@ class ProcessPoolExecutor(_process.ProcessPoolExecutor): super(ProcessPoolExecutor, self).__init__(max_workers=max_workers) if self._max_workers <= 0: raise ValueError("Max workers must be greater than zero") + self._gatherer = _Gatherer( + # Since our submit will use this gatherer we have to reference + # the parent submit, bound to this instance (which is what we + # really want to use anyway). + super(ProcessPoolExecutor, self).submit) @property def alive(self): return not self._shutdown_thread + @property + def statistics(self): + """:class:`.ExecutorStatistics` about the executors executions.""" + return self._gatherer.statistics + + def submit(self, fn, *args, **kwargs): + """Submit some work to be executed (and gather statistics).""" + return self._gatherer.submit(fn, *args, **kwargs) + class _WorkItem(object): def __init__(self, future, fn, args, kwargs): @@ -93,10 +178,14 @@ class SynchronousExecutor(_futures.Executor): will execute the calls inside the caller thread instead of executing it in a external process/thread for when this type of functionality is useful to provide... + + It gathers statistics about the submissions executed for post-analysis... """ def __init__(self): self._shutoff = False + self._gatherer = _Gatherer(self._submit, + start_before_submit=True) @property def alive(self): @@ -105,10 +194,19 @@ class SynchronousExecutor(_futures.Executor): def shutdown(self, wait=True): self._shutoff = True + @property + def statistics(self): + """:class:`.ExecutorStatistics` about the executors executions.""" + return self._gatherer.statistics + def submit(self, fn, *args, **kwargs): + """Submit some work to be executed (and gather statistics).""" if self._shutoff: raise RuntimeError('Can not schedule new futures' ' after being shutdown') + return self._gatherer.submit(fn, *args, **kwargs) + + def _submit(self, fn, *args, **kwargs): f = Future() runner = _WorkItem(f, fn, args, kwargs) runner.run() @@ -160,6 +258,8 @@ class GreenThreadPoolExecutor(_futures.Executor): See: https://docs.python.org/dev/library/concurrent.futures.html and http://eventlet.net/doc/modules/greenpool.html for information on how this works. + + It gathers statistics about the submissions executed for post-analysis... """ def __init__(self, max_workers=1000): @@ -171,21 +271,32 @@ class GreenThreadPoolExecutor(_futures.Executor): self._delayed_work = greenqueue.Queue() self._shutdown_lock = greenthreading.Lock() self._shutdown = False + self._gatherer = _Gatherer(self._submit, + lock_cls=greenthreading.Lock) @property def alive(self): return not self._shutdown + @property + def statistics(self): + """:class:`.ExecutorStatistics` about the executors executions.""" + return self._gatherer.statistics + def submit(self, fn, *args, **kwargs): + """Submit some work to be executed (and gather statistics).""" with self._shutdown_lock: if self._shutdown: raise RuntimeError('Can not schedule new futures' ' after being shutdown') - f = GreenFuture() - work = _WorkItem(f, fn, args, kwargs) - if not self._spin_up(work): - self._delayed_work.put(work) - return f + return self._gatherer.submit(fn, *args, **kwargs) + + def _submit(self, fn, *args, **kwargs): + f = GreenFuture() + work = _WorkItem(f, fn, args, kwargs) + if not self._spin_up(work): + self._delayed_work.put(work) + return f def _spin_up(self, work): alive = self._pool.running() + self._pool.waiting() @@ -204,3 +315,58 @@ class GreenThreadPoolExecutor(_futures.Executor): if wait and shutoff: self._pool.waitall() self._delayed_work.join() + + +class ExecutorStatistics(object): + """Holds *immutable* information about a executors executions.""" + + __slots__ = ['_failures', '_executed', '_runtime', '_cancelled'] + + __repr_format = ("failures=%(failures)s, executed=%(executed)s, " + "runtime=%(runtime)s, cancelled=%(cancelled)s") + + def __init__(self, failures=0, executed=0, runtime=0.0, cancelled=0): + self._failures = failures + self._executed = executed + self._runtime = runtime + self._cancelled = cancelled + + @property + def failures(self): + """How many submissions ended up raising exceptions.""" + return self._failures + + @property + def executed(self): + """How many submissions were executed (failed or not).""" + return self._executed + + @property + def runtime(self): + """Total runtime of all submissions executed.""" + return self._runtime + + @property + def cancelled(self): + """How many submissions were cancelled before executing.""" + return self._cancelled + + @property + def average_runtime(self): + """The average runtime of all submissions executed. + + :raises: ZeroDivisionError when no executions have occurred. + """ + return self._runtime / self._executed + + def __repr__(self): + r = reflection.get_class_name(self, fully_qualified=False) + r += "(" + r += self.__repr_format % ({ + 'failures': self._failures, + 'executed': self._executed, + 'runtime': self._runtime, + 'cancelled': self._cancelled, + }) + r += ")" + return r From d748db934a0348da3f1f5b86e61e0b5ca9d2b69a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 10 Jan 2015 20:34:10 -0800 Subject: [PATCH 197/240] Switch to using 'oslo_utils' vs 'oslo.utils' Prefer the non-deprecated 'oslo_utils' instead of the namespaced 'oslo.utils' wherever it was previously used. Change-Id: I9a78150ef5266e1ff22147278162fe3cfe1b2e3f --- taskflow/atom.py | 2 +- taskflow/engines/action_engine/engine.py | 2 +- taskflow/engines/action_engine/executor.py | 8 ++++---- taskflow/engines/helpers.py | 4 ++-- taskflow/engines/worker_based/endpoint.py | 2 +- taskflow/engines/worker_based/executor.py | 4 ++-- taskflow/engines/worker_based/protocol.py | 4 ++-- taskflow/engines/worker_based/worker.py | 2 +- taskflow/examples/create_parallel_volume.py | 2 +- taskflow/examples/fake_billing.py | 2 +- taskflow/examples/resume_vm_boot.py | 2 +- taskflow/flow.py | 2 +- taskflow/jobs/backends/impl_zookeeper.py | 4 ++-- taskflow/jobs/job.py | 2 +- taskflow/listeners/base.py | 2 +- taskflow/persistence/backends/impl_sqlalchemy.py | 2 +- taskflow/persistence/backends/sqlalchemy/models.py | 4 ++-- taskflow/persistence/logbook.py | 4 ++-- taskflow/storage.py | 4 ++-- taskflow/task.py | 2 +- taskflow/tests/unit/jobs/base.py | 2 +- taskflow/tests/unit/jobs/test_zk_job.py | 2 +- taskflow/tests/unit/persistence/base.py | 2 +- taskflow/tests/unit/persistence/test_zk_persistence.py | 2 +- taskflow/tests/unit/test_engine_helpers.py | 4 ++-- taskflow/tests/unit/test_listeners.py | 2 +- taskflow/tests/unit/test_storage.py | 2 +- taskflow/tests/unit/test_types.py | 2 +- taskflow/tests/unit/worker_based/test_endpoint.py | 2 +- taskflow/tests/unit/worker_based/test_executor.py | 2 +- taskflow/tests/unit/worker_based/test_message_pump.py | 2 +- taskflow/tests/unit/worker_based/test_pipeline.py | 2 +- taskflow/tests/unit/worker_based/test_protocol.py | 4 ++-- taskflow/tests/unit/worker_based/test_worker.py | 2 +- taskflow/types/cache.py | 2 +- taskflow/types/failure.py | 2 +- taskflow/types/notifier.py | 2 +- taskflow/types/timing.py | 4 ++-- taskflow/utils/deprecation.py | 2 +- taskflow/utils/kazoo_utils.py | 2 +- taskflow/utils/misc.py | 6 +++--- taskflow/utils/persistence_utils.py | 4 ++-- 42 files changed, 58 insertions(+), 58 deletions(-) diff --git a/taskflow/atom.py b/taskflow/atom.py index 3131664f..a2066048 100644 --- a/taskflow/atom.py +++ b/taskflow/atom.py @@ -15,7 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo.utils import reflection +from oslo_utils import reflection import six from taskflow import exceptions diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index 157f641d..ed06b64f 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -19,7 +19,7 @@ import contextlib import threading from concurrent import futures -from oslo.utils import excutils +from oslo_utils import excutils import six from taskflow.engines.action_engine import compiler diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index 1c925678..7f6c3ff6 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -20,10 +20,10 @@ from multiprocessing import managers import os import pickle -from oslo.utils import excutils -from oslo.utils import reflection -from oslo.utils import timeutils -from oslo.utils import uuidutils +from oslo_utils import excutils +from oslo_utils import reflection +from oslo_utils import timeutils +from oslo_utils import uuidutils import six from six.moves import queue as compat_queue diff --git a/taskflow/engines/helpers.py b/taskflow/engines/helpers.py index 2c84a3c3..3d8ccf55 100644 --- a/taskflow/engines/helpers.py +++ b/taskflow/engines/helpers.py @@ -18,8 +18,8 @@ import contextlib import itertools import traceback -from oslo.utils import importutils -from oslo.utils import reflection +from oslo_utils import importutils +from oslo_utils import reflection import six import stevedore.driver diff --git a/taskflow/engines/worker_based/endpoint.py b/taskflow/engines/worker_based/endpoint.py index 0f883ade..2c85310e 100644 --- a/taskflow/engines/worker_based/endpoint.py +++ b/taskflow/engines/worker_based/endpoint.py @@ -14,7 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo.utils import reflection +from oslo_utils import reflection from taskflow.engines.action_engine import executor diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index aea63cfc..b07c73f8 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -17,8 +17,8 @@ import functools import threading -from oslo.utils import reflection -from oslo.utils import timeutils +from oslo_utils import reflection +from oslo_utils import timeutils from taskflow.engines.action_engine import executor from taskflow.engines.worker_based import cache diff --git a/taskflow/engines/worker_based/protocol.py b/taskflow/engines/worker_based/protocol.py index 96ba84c4..19813d40 100644 --- a/taskflow/engines/worker_based/protocol.py +++ b/taskflow/engines/worker_based/protocol.py @@ -20,8 +20,8 @@ import threading from concurrent import futures import jsonschema from jsonschema import exceptions as schema_exc -from oslo.utils import reflection -from oslo.utils import timeutils +from oslo_utils import reflection +from oslo_utils import timeutils import six from taskflow.engines.action_engine import executor diff --git a/taskflow/engines/worker_based/worker.py b/taskflow/engines/worker_based/worker.py index 5464c814..98e690ef 100644 --- a/taskflow/engines/worker_based/worker.py +++ b/taskflow/engines/worker_based/worker.py @@ -21,7 +21,7 @@ import string import sys from concurrent import futures -from oslo.utils import reflection +from oslo_utils import reflection from taskflow.engines.worker_based import endpoint from taskflow.engines.worker_based import server diff --git a/taskflow/examples/create_parallel_volume.py b/taskflow/examples/create_parallel_volume.py index 0416a25e..c23bf342 100644 --- a/taskflow/examples/create_parallel_volume.py +++ b/taskflow/examples/create_parallel_volume.py @@ -28,7 +28,7 @@ top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) sys.path.insert(0, top_dir) -from oslo.utils import reflection +from oslo_utils import reflection from taskflow import engines from taskflow.listeners import printing diff --git a/taskflow/examples/fake_billing.py b/taskflow/examples/fake_billing.py index 0fbe81f7..8e0c181e 100644 --- a/taskflow/examples/fake_billing.py +++ b/taskflow/examples/fake_billing.py @@ -27,7 +27,7 @@ top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) sys.path.insert(0, top_dir) -from oslo.utils import uuidutils +from oslo_utils import uuidutils from taskflow import engines from taskflow.listeners import printing diff --git a/taskflow/examples/resume_vm_boot.py b/taskflow/examples/resume_vm_boot.py index f400d0d1..9c119fc2 100644 --- a/taskflow/examples/resume_vm_boot.py +++ b/taskflow/examples/resume_vm_boot.py @@ -31,7 +31,7 @@ top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), sys.path.insert(0, top_dir) sys.path.insert(0, self_dir) -from oslo.utils import uuidutils +from oslo_utils import uuidutils from taskflow import engines from taskflow import exceptions as exc diff --git a/taskflow/flow.py b/taskflow/flow.py index 683cf8b4..cd70e7c9 100644 --- a/taskflow/flow.py +++ b/taskflow/flow.py @@ -16,7 +16,7 @@ import abc -from oslo.utils import reflection +from oslo_utils import reflection import six # Link metadata keys that have inherent/special meaning. diff --git a/taskflow/jobs/backends/impl_zookeeper.py b/taskflow/jobs/backends/impl_zookeeper.py index 65f88902..186be8e6 100644 --- a/taskflow/jobs/backends/impl_zookeeper.py +++ b/taskflow/jobs/backends/impl_zookeeper.py @@ -24,8 +24,8 @@ from kazoo import exceptions as k_exceptions from kazoo.protocol import paths as k_paths from kazoo.recipe import watchers from oslo.serialization import jsonutils -from oslo.utils import excutils -from oslo.utils import uuidutils +from oslo_utils import excutils +from oslo_utils import uuidutils import six from taskflow import exceptions as excp diff --git a/taskflow/jobs/job.py b/taskflow/jobs/job.py index 23e33ee7..a7dd04f7 100644 --- a/taskflow/jobs/job.py +++ b/taskflow/jobs/job.py @@ -17,7 +17,7 @@ import abc -from oslo.utils import uuidutils +from oslo_utils import uuidutils import six diff --git a/taskflow/listeners/base.py b/taskflow/listeners/base.py index 42dc87e9..182a7ddb 100644 --- a/taskflow/listeners/base.py +++ b/taskflow/listeners/base.py @@ -18,7 +18,7 @@ from __future__ import absolute_import import abc -from oslo.utils import excutils +from oslo_utils import excutils import six from taskflow import logging diff --git a/taskflow/persistence/backends/impl_sqlalchemy.py b/taskflow/persistence/backends/impl_sqlalchemy.py index c65c9134..d84b9b27 100644 --- a/taskflow/persistence/backends/impl_sqlalchemy.py +++ b/taskflow/persistence/backends/impl_sqlalchemy.py @@ -24,7 +24,7 @@ import copy import functools import time -from oslo.utils import strutils +from oslo_utils import strutils import six import sqlalchemy as sa from sqlalchemy import exc as sa_exc diff --git a/taskflow/persistence/backends/sqlalchemy/models.py b/taskflow/persistence/backends/sqlalchemy/models.py index 3f056de5..e6a55768 100644 --- a/taskflow/persistence/backends/sqlalchemy/models.py +++ b/taskflow/persistence/backends/sqlalchemy/models.py @@ -16,8 +16,8 @@ # under the License. from oslo.serialization import jsonutils -from oslo.utils import timeutils -from oslo.utils import uuidutils +from oslo_utils import timeutils +from oslo_utils import uuidutils from sqlalchemy import Column, String, DateTime, Enum from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import ForeignKey diff --git a/taskflow/persistence/logbook.py b/taskflow/persistence/logbook.py index 284fbf66..9ef3efec 100644 --- a/taskflow/persistence/logbook.py +++ b/taskflow/persistence/logbook.py @@ -18,8 +18,8 @@ import abc import copy -from oslo.utils import timeutils -from oslo.utils import uuidutils +from oslo_utils import timeutils +from oslo_utils import uuidutils import six from taskflow import exceptions as exc diff --git a/taskflow/storage.py b/taskflow/storage.py index e698ca4d..8cb81f3b 100644 --- a/taskflow/storage.py +++ b/taskflow/storage.py @@ -17,8 +17,8 @@ import abc import contextlib -from oslo.utils import reflection -from oslo.utils import uuidutils +from oslo_utils import reflection +from oslo_utils import uuidutils import six from taskflow import exceptions diff --git a/taskflow/task.py b/taskflow/task.py index 7a1c7180..8fa9ffb5 100644 --- a/taskflow/task.py +++ b/taskflow/task.py @@ -18,7 +18,7 @@ import abc import copy -from oslo.utils import reflection +from oslo_utils import reflection import six from taskflow import atom diff --git a/taskflow/tests/unit/jobs/base.py b/taskflow/tests/unit/jobs/base.py index 4c070f31..8a8bee22 100644 --- a/taskflow/tests/unit/jobs/base.py +++ b/taskflow/tests/unit/jobs/base.py @@ -18,7 +18,7 @@ import contextlib import time from kazoo.recipe import watchers -from oslo.utils import uuidutils +from oslo_utils import uuidutils from taskflow import exceptions as excp from taskflow.persistence.backends import impl_dir diff --git a/taskflow/tests/unit/jobs/test_zk_job.py b/taskflow/tests/unit/jobs/test_zk_job.py index 8737a4f5..8b52db6a 100644 --- a/taskflow/tests/unit/jobs/test_zk_job.py +++ b/taskflow/tests/unit/jobs/test_zk_job.py @@ -15,7 +15,7 @@ # under the License. from oslo.serialization import jsonutils -from oslo.utils import uuidutils +from oslo_utils import uuidutils import six import testtools from zake import fake_client diff --git a/taskflow/tests/unit/persistence/base.py b/taskflow/tests/unit/persistence/base.py index 88660fd5..184cf51e 100644 --- a/taskflow/tests/unit/persistence/base.py +++ b/taskflow/tests/unit/persistence/base.py @@ -16,7 +16,7 @@ import contextlib -from oslo.utils import uuidutils +from oslo_utils import uuidutils from taskflow import exceptions as exc from taskflow.persistence import logbook diff --git a/taskflow/tests/unit/persistence/test_zk_persistence.py b/taskflow/tests/unit/persistence/test_zk_persistence.py index 28463bb7..bb8bec9a 100644 --- a/taskflow/tests/unit/persistence/test_zk_persistence.py +++ b/taskflow/tests/unit/persistence/test_zk_persistence.py @@ -17,7 +17,7 @@ import contextlib from kazoo import exceptions as kazoo_exceptions -from oslo.utils import uuidutils +from oslo_utils import uuidutils import testtools from zake import fake_client diff --git a/taskflow/tests/unit/test_engine_helpers.py b/taskflow/tests/unit/test_engine_helpers.py index 4087d839..ed40caa0 100644 --- a/taskflow/tests/unit/test_engine_helpers.py +++ b/taskflow/tests/unit/test_engine_helpers.py @@ -81,7 +81,7 @@ class FlowFromDetailTestCase(test.TestCase): _lb, flow_detail = p_utils.temporary_flow_detail() flow_detail.meta = dict(factory=dict(name=name)) - with mock.patch('oslo.utils.importutils.import_class', + with mock.patch('oslo_utils.importutils.import_class', return_value=lambda: 'RESULT') as mock_import: result = taskflow.engines.flow_from_detail(flow_detail) mock_import.assert_called_onec_with(name) @@ -92,7 +92,7 @@ class FlowFromDetailTestCase(test.TestCase): _lb, flow_detail = p_utils.temporary_flow_detail() flow_detail.meta = dict(factory=dict(name=name, args=['foo'])) - with mock.patch('oslo.utils.importutils.import_class', + with mock.patch('oslo_utils.importutils.import_class', return_value=lambda x: 'RESULT %s' % x) as mock_import: result = taskflow.engines.flow_from_detail(flow_detail) mock_import.assert_called_onec_with(name) diff --git a/taskflow/tests/unit/test_listeners.py b/taskflow/tests/unit/test_listeners.py index c10bc282..625b9a1f 100644 --- a/taskflow/tests/unit/test_listeners.py +++ b/taskflow/tests/unit/test_listeners.py @@ -19,7 +19,7 @@ import logging import time from oslo.serialization import jsonutils -from oslo.utils import reflection +from oslo_utils import reflection import six from zake import fake_client diff --git a/taskflow/tests/unit/test_storage.py b/taskflow/tests/unit/test_storage.py index 886e075a..5f521afb 100644 --- a/taskflow/tests/unit/test_storage.py +++ b/taskflow/tests/unit/test_storage.py @@ -17,7 +17,7 @@ import contextlib import threading -from oslo.utils import uuidutils +from oslo_utils import uuidutils from taskflow import exceptions from taskflow.persistence import backends diff --git a/taskflow/tests/unit/test_types.py b/taskflow/tests/unit/test_types.py index 85d6e557..8aae36fa 100644 --- a/taskflow/tests/unit/test_types.py +++ b/taskflow/tests/unit/test_types.py @@ -15,7 +15,7 @@ # under the License. import networkx as nx -from oslo.utils import timeutils +from oslo_utils import timeutils import six from taskflow import exceptions as excp diff --git a/taskflow/tests/unit/worker_based/test_endpoint.py b/taskflow/tests/unit/worker_based/test_endpoint.py index 53260b12..6f13c8be 100644 --- a/taskflow/tests/unit/worker_based/test_endpoint.py +++ b/taskflow/tests/unit/worker_based/test_endpoint.py @@ -14,7 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo.utils import reflection +from oslo_utils import reflection from taskflow.engines.worker_based import endpoint as ep from taskflow import task diff --git a/taskflow/tests/unit/worker_based/test_executor.py b/taskflow/tests/unit/worker_based/test_executor.py index f075e2f8..4e0c38bc 100644 --- a/taskflow/tests/unit/worker_based/test_executor.py +++ b/taskflow/tests/unit/worker_based/test_executor.py @@ -17,7 +17,7 @@ import time from concurrent import futures -from oslo.utils import timeutils +from oslo_utils import timeutils from taskflow.engines.worker_based import executor from taskflow.engines.worker_based import protocol as pr diff --git a/taskflow/tests/unit/worker_based/test_message_pump.py b/taskflow/tests/unit/worker_based/test_message_pump.py index 7b945a26..d8438131 100644 --- a/taskflow/tests/unit/worker_based/test_message_pump.py +++ b/taskflow/tests/unit/worker_based/test_message_pump.py @@ -14,7 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo.utils import uuidutils +from oslo_utils import uuidutils from taskflow.engines.worker_based import protocol as pr from taskflow.engines.worker_based import proxy diff --git a/taskflow/tests/unit/worker_based/test_pipeline.py b/taskflow/tests/unit/worker_based/test_pipeline.py index 53bf8f9b..8d4de7f5 100644 --- a/taskflow/tests/unit/worker_based/test_pipeline.py +++ b/taskflow/tests/unit/worker_based/test_pipeline.py @@ -15,7 +15,7 @@ # under the License. from concurrent import futures -from oslo.utils import uuidutils +from oslo_utils import uuidutils from taskflow.engines.action_engine import executor as base_executor from taskflow.engines.worker_based import endpoint diff --git a/taskflow/tests/unit/worker_based/test_protocol.py b/taskflow/tests/unit/worker_based/test_protocol.py index 4c34ed60..5356fb99 100644 --- a/taskflow/tests/unit/worker_based/test_protocol.py +++ b/taskflow/tests/unit/worker_based/test_protocol.py @@ -15,8 +15,8 @@ # under the License. from concurrent import futures -from oslo.utils import timeutils -from oslo.utils import uuidutils +from oslo_utils import timeutils +from oslo_utils import uuidutils from taskflow.engines.action_engine import executor from taskflow.engines.worker_based import protocol as pr diff --git a/taskflow/tests/unit/worker_based/test_worker.py b/taskflow/tests/unit/worker_based/test_worker.py index 8fc76eb4..4bf86529 100644 --- a/taskflow/tests/unit/worker_based/test_worker.py +++ b/taskflow/tests/unit/worker_based/test_worker.py @@ -14,7 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo.utils import reflection +from oslo_utils import reflection import six from taskflow.engines.worker_based import endpoint diff --git a/taskflow/types/cache.py b/taskflow/types/cache.py index c3ac7a18..d9b14910 100644 --- a/taskflow/types/cache.py +++ b/taskflow/types/cache.py @@ -16,7 +16,7 @@ import threading -from oslo.utils import reflection +from oslo_utils import reflection import six diff --git a/taskflow/types/failure.py b/taskflow/types/failure.py index ae848d31..4732fb14 100644 --- a/taskflow/types/failure.py +++ b/taskflow/types/failure.py @@ -19,7 +19,7 @@ import os import sys import traceback -from oslo.utils import reflection +from oslo_utils import reflection import six from taskflow import exceptions as exc diff --git a/taskflow/types/notifier.py b/taskflow/types/notifier.py index 838585a2..9f4df801 100644 --- a/taskflow/types/notifier.py +++ b/taskflow/types/notifier.py @@ -19,7 +19,7 @@ import contextlib import copy import logging -from oslo.utils import reflection +from oslo_utils import reflection import six LOG = logging.getLogger(__name__) diff --git a/taskflow/types/timing.py b/taskflow/types/timing.py index ab1d39dd..e7fa7d45 100644 --- a/taskflow/types/timing.py +++ b/taskflow/types/timing.py @@ -14,8 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo.utils import reflection -from oslo.utils import timeutils +from oslo_utils import reflection +from oslo_utils import timeutils from taskflow.utils import threading_utils diff --git a/taskflow/utils/deprecation.py b/taskflow/utils/deprecation.py index 5c466662..60f82d6f 100644 --- a/taskflow/utils/deprecation.py +++ b/taskflow/utils/deprecation.py @@ -17,7 +17,7 @@ import functools import warnings -from oslo.utils import reflection +from oslo_utils import reflection import six _CLASS_MOVED_PREFIX_TPL = "Class '%s' has moved to '%s'" diff --git a/taskflow/utils/kazoo_utils.py b/taskflow/utils/kazoo_utils.py index ab449635..35fe058e 100644 --- a/taskflow/utils/kazoo_utils.py +++ b/taskflow/utils/kazoo_utils.py @@ -16,7 +16,7 @@ from kazoo import client from kazoo import exceptions as k_exc -from oslo.utils import reflection +from oslo_utils import reflection import six from six.moves import zip as compat_zip diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 5aa62fd9..04150003 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -26,9 +26,9 @@ import threading import types from oslo.serialization import jsonutils -from oslo.utils import importutils -from oslo.utils import netutils -from oslo.utils import reflection +from oslo_utils import importutils +from oslo_utils import netutils +from oslo_utils import reflection import six from six.moves import map as compat_map from six.moves import range as compat_range diff --git a/taskflow/utils/persistence_utils.py b/taskflow/utils/persistence_utils.py index 6d5d1685..dd304bc6 100644 --- a/taskflow/utils/persistence_utils.py +++ b/taskflow/utils/persistence_utils.py @@ -17,8 +17,8 @@ import contextlib import os -from oslo.utils import timeutils -from oslo.utils import uuidutils +from oslo_utils import timeutils +from oslo_utils import uuidutils from taskflow import logging from taskflow.persistence import logbook From 2280f9a7485e5a403a4307dd1f3aeb3ace1e99f9 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 10 Jan 2015 20:36:44 -0800 Subject: [PATCH 198/240] Switch to using 'oslo_serialization' vs 'oslo.serialization' Prefer the non-deprecated 'oslo_serialization' instead of the namespaced 'oslo.serialization' wherever it was previously used. Change-Id: I652cf0b56e28d727c59fe0c060949bb2bd920d11 --- taskflow/jobs/backends/impl_zookeeper.py | 2 +- taskflow/persistence/backends/impl_dir.py | 2 +- taskflow/persistence/backends/impl_zookeeper.py | 2 +- taskflow/persistence/backends/sqlalchemy/models.py | 2 +- taskflow/tests/unit/jobs/test_zk_job.py | 2 +- taskflow/tests/unit/test_listeners.py | 2 +- taskflow/utils/misc.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/taskflow/jobs/backends/impl_zookeeper.py b/taskflow/jobs/backends/impl_zookeeper.py index 186be8e6..479d0732 100644 --- a/taskflow/jobs/backends/impl_zookeeper.py +++ b/taskflow/jobs/backends/impl_zookeeper.py @@ -23,7 +23,7 @@ from concurrent import futures from kazoo import exceptions as k_exceptions from kazoo.protocol import paths as k_paths from kazoo.recipe import watchers -from oslo.serialization import jsonutils +from oslo_serialization import jsonutils from oslo_utils import excutils from oslo_utils import uuidutils import six diff --git a/taskflow/persistence/backends/impl_dir.py b/taskflow/persistence/backends/impl_dir.py index 0a687473..1f10b925 100644 --- a/taskflow/persistence/backends/impl_dir.py +++ b/taskflow/persistence/backends/impl_dir.py @@ -19,7 +19,7 @@ import errno import os import shutil -from oslo.serialization import jsonutils +from oslo_serialization import jsonutils import six from taskflow import exceptions as exc diff --git a/taskflow/persistence/backends/impl_zookeeper.py b/taskflow/persistence/backends/impl_zookeeper.py index ca801b43..11024212 100644 --- a/taskflow/persistence/backends/impl_zookeeper.py +++ b/taskflow/persistence/backends/impl_zookeeper.py @@ -18,7 +18,7 @@ import contextlib from kazoo import exceptions as k_exc from kazoo.protocol import paths -from oslo.serialization import jsonutils +from oslo_serialization import jsonutils from taskflow import exceptions as exc from taskflow import logging diff --git a/taskflow/persistence/backends/sqlalchemy/models.py b/taskflow/persistence/backends/sqlalchemy/models.py index e6a55768..e43c9652 100644 --- a/taskflow/persistence/backends/sqlalchemy/models.py +++ b/taskflow/persistence/backends/sqlalchemy/models.py @@ -15,7 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo.serialization import jsonutils +from oslo_serialization import jsonutils from oslo_utils import timeutils from oslo_utils import uuidutils from sqlalchemy import Column, String, DateTime, Enum diff --git a/taskflow/tests/unit/jobs/test_zk_job.py b/taskflow/tests/unit/jobs/test_zk_job.py index 8b52db6a..afff4123 100644 --- a/taskflow/tests/unit/jobs/test_zk_job.py +++ b/taskflow/tests/unit/jobs/test_zk_job.py @@ -14,7 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo.serialization import jsonutils +from oslo_serialization import jsonutils from oslo_utils import uuidutils import six import testtools diff --git a/taskflow/tests/unit/test_listeners.py b/taskflow/tests/unit/test_listeners.py index 625b9a1f..d6c64bbd 100644 --- a/taskflow/tests/unit/test_listeners.py +++ b/taskflow/tests/unit/test_listeners.py @@ -18,7 +18,7 @@ import contextlib import logging import time -from oslo.serialization import jsonutils +from oslo_serialization import jsonutils from oslo_utils import reflection import six from zake import fake_client diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 04150003..63a7598f 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -25,7 +25,7 @@ import sys import threading import types -from oslo.serialization import jsonutils +from oslo_serialization import jsonutils from oslo_utils import importutils from oslo_utils import netutils from oslo_utils import reflection From 39685086165e459e6e644b011e5d2417f0c17098 Mon Sep 17 00:00:00 2001 From: Changbin Liu Date: Thu, 15 Jan 2015 12:08:10 -0500 Subject: [PATCH 199/240] Fix unused and conflicting variables Prefix '_' to denote unused variables Suffix '_' to avoid conflicting with Python built-in funcs Change-Id: I4e0d0b8f88e5c93222fbd7f8dc7cf626923f738e --- taskflow/engines/action_engine/compiler.py | 2 +- taskflow/examples/fake_billing.py | 6 +++--- taskflow/examples/resume_many_flows.py | 2 +- taskflow/examples/resume_vm_boot.py | 2 +- taskflow/examples/wbe_mandelbrot.py | 2 +- taskflow/types/fsm.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/taskflow/engines/action_engine/compiler.py b/taskflow/engines/action_engine/compiler.py index c55a1e7f..fb81ba80 100644 --- a/taskflow/engines/action_engine/compiler.py +++ b/taskflow/engines/action_engine/compiler.py @@ -342,7 +342,7 @@ class PatternCompiler(object): node.add(tr.Node(flow.retry)) decomposed_members = {} for item in flow: - subgraph, subnode = self._flatten(item, node) + subgraph, _subnode = self._flatten(item, node) decomposed_members[item] = subgraph if subgraph.number_of_nodes(): graph = gr.merge_graphs([graph, subgraph]) diff --git a/taskflow/examples/fake_billing.py b/taskflow/examples/fake_billing.py index 0fbe81f7..5889a2cd 100644 --- a/taskflow/examples/fake_billing.py +++ b/taskflow/examples/fake_billing.py @@ -149,9 +149,9 @@ class DeclareSuccess(task.Task): class DummyUser(object): - def __init__(self, user, id): + def __init__(self, user, id_): self.user = user - self.id = id + self.id = id_ # Resources (db handles and similar) of course can *not* be persisted so we @@ -174,7 +174,7 @@ flow.add(sub_flow) # prepopulating this allows the tasks that dependent on the 'request' variable # to start processing (in this case this is the ExtractInputRequest task). store = { - 'request': DummyUser(user="bob", id="1.35"), + 'request': DummyUser(user="bob", id_="1.35"), } eng = engines.load(flow, engine='serial', store=store) diff --git a/taskflow/examples/resume_many_flows.py b/taskflow/examples/resume_many_flows.py index 08cc1740..88e55510 100644 --- a/taskflow/examples/resume_many_flows.py +++ b/taskflow/examples/resume_many_flows.py @@ -48,7 +48,7 @@ def _exec(cmd, add_env=None): stdout=subprocess.PIPE, stderr=sys.stderr) - stdout, stderr = proc.communicate() + stdout, _stderr = proc.communicate() rc = proc.returncode if rc != 0: raise RuntimeError("Could not run %s [%s]", cmd, rc) diff --git a/taskflow/examples/resume_vm_boot.py b/taskflow/examples/resume_vm_boot.py index f400d0d1..50714ec1 100644 --- a/taskflow/examples/resume_vm_boot.py +++ b/taskflow/examples/resume_vm_boot.py @@ -143,7 +143,7 @@ class AllocateIP(task.Task): def execute(self, vm_spec): ips = [] - for i in range(0, vm_spec.get('ips', 0)): + for _i in range(0, vm_spec.get('ips', 0)): ips.append("192.168.0.%s" % (random.randint(1, 254))) return ips diff --git a/taskflow/examples/wbe_mandelbrot.py b/taskflow/examples/wbe_mandelbrot.py index f21465e3..c59b85ce 100644 --- a/taskflow/examples/wbe_mandelbrot.py +++ b/taskflow/examples/wbe_mandelbrot.py @@ -131,7 +131,7 @@ def calculate(engine_conf): task_names = [] # Compose our workflow. - height, width = IMAGE_SIZE + height, _width = IMAGE_SIZE chunk_size = int(math.ceil(height / float(CHUNK_COUNT))) for i in compat_range(0, CHUNK_COUNT): chunk_name = 'chunk_%s' % i diff --git a/taskflow/types/fsm.py b/taskflow/types/fsm.py index 2519e840..6ca22909 100644 --- a/taskflow/types/fsm.py +++ b/taskflow/types/fsm.py @@ -205,7 +205,7 @@ class FSM(object): def run(self, event, initialize=True): """Runs the state machine, using reactions only.""" - for transition in self.run_iter(event, initialize=initialize): + for _transition in self.run_iter(event, initialize=initialize): pass def copy(self): From 22eef9618b7e84a4dea7bb143f36ed7e0fc11924 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 15 Jan 2015 11:29:33 -0800 Subject: [PATCH 200/240] Provide the stopwatch elapsed method a maximum To anticipate moving this code over to oslo.utils so that it can be shared by taskflow, oslo.messaging (and others) add a maximum elapsed value to the elapsed method which will limit the maximum value it returns. Change-Id: Ica85cbd39b34211ac04cd58dcbdc499a778487b0 --- taskflow/tests/unit/test_types.py | 17 +++++++++++++++++ taskflow/types/timing.py | 19 +++++++++++-------- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/taskflow/tests/unit/test_types.py b/taskflow/tests/unit/test_types.py index 85d6e557..d13124ac 100644 --- a/taskflow/tests/unit/test_types.py +++ b/taskflow/tests/unit/test_types.py @@ -217,6 +217,23 @@ class StopWatchTest(test.TestCase): watch.start() self.assertEqual(0, len(watch.splits)) + def test_elapsed_maximum(self): + watch = tt.StopWatch() + watch.start() + + timeutils.advance_time_seconds(1) + self.assertEqual(1, watch.elapsed()) + + timeutils.advance_time_seconds(10) + self.assertEqual(11, watch.elapsed()) + self.assertEqual(1, watch.elapsed(maximum=1)) + + watch.stop() + self.assertEqual(11, watch.elapsed()) + timeutils.advance_time_seconds(10) + self.assertEqual(11, watch.elapsed()) + self.assertEqual(0, watch.elapsed(maximum=-1)) + class TableTest(test.TestCase): def test_create_valid_no_rows(self): diff --git a/taskflow/types/timing.py b/taskflow/types/timing.py index 2cbeee9b..5001762f 100644 --- a/taskflow/types/timing.py +++ b/taskflow/types/timing.py @@ -136,17 +136,20 @@ class StopWatch(object): self.start() return self - def elapsed(self): + def elapsed(self, maximum=None): """Returns how many seconds have elapsed.""" - if self._state == self._STOPPED: - return max(0.0, float(timeutils.delta_seconds(self._started_at, - self._stopped_at))) - elif self._state == self._STARTED: - return max(0.0, float(timeutils.delta_seconds(self._started_at, - timeutils.utcnow()))) - else: + if self._state not in (self._STOPPED, self._STARTED): raise RuntimeError("Can not get the elapsed time of a stopwatch" " if it has not been started/stopped") + if self._state == self._STOPPED: + elapsed = max(0.0, float(timeutils.delta_seconds( + self._started_at, self._stopped_at))) + else: + elapsed = max(0.0, float(timeutils.delta_seconds( + self._started_at, timeutils.utcnow()))) + if maximum is not None and elapsed > maximum: + elapsed = max(0.0, maximum) + return elapsed def __enter__(self): """Starts the watch.""" From bfc11369f0c47cb79e895a7d4de3808d36f2219e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 13 Jan 2015 16:20:56 -0800 Subject: [PATCH 201/240] Remove 'SaveOrderTask' and test state in class variables Instead of saving task state in a class variable that is later introspected by further test code just remove that concept (which doesn't work in multiprocessing or worker engines which can not have access those types of shared/globally available concepts due to how they run) and use a specialized listener that can gather the same information in a more decoupled manner (and it will work in multiprocessing and worker engines correctly). This allows our engine test cases to work in those engine types which increases those engines test coverage (and future coverage and engine tests that are added). Fixes a bunch of occurrences of bug 1357117 as well that were removed during this cleanup and adjustment process... Change-Id: Ic9901de2902ac28ec255bef146be5846d18f9bfb --- .../engines/action_engine/actions/base.py | 42 + .../engines/action_engine/actions/retry.py | 21 +- .../engines/action_engine/actions/task.py | 30 +- taskflow/engines/action_engine/runtime.py | 5 +- taskflow/listeners/base.py | 1 + .../tests/unit/conductor/test_conductor.py | 2 +- taskflow/tests/unit/test_engines.py | 475 +++---- taskflow/tests/unit/test_retries.py | 1089 ++++++++++------- taskflow/tests/unit/test_suspend.py | 237 ++++ taskflow/tests/unit/test_suspend_flow.py | 195 --- .../tests/unit/worker_based/test_worker.py | 2 +- taskflow/tests/utils.py | 88 +- taskflow/types/futures.py | 4 + 13 files changed, 1293 insertions(+), 898 deletions(-) create mode 100644 taskflow/engines/action_engine/actions/base.py create mode 100644 taskflow/tests/unit/test_suspend.py delete mode 100644 taskflow/tests/unit/test_suspend_flow.py diff --git a/taskflow/engines/action_engine/actions/base.py b/taskflow/engines/action_engine/actions/base.py new file mode 100644 index 00000000..5595268a --- /dev/null +++ b/taskflow/engines/action_engine/actions/base.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc + +import six + +from taskflow import states + + +#: Sentinel use to represent no-result (none can be a valid result...) +NO_RESULT = object() + +#: States that are expected to/may have a result to save... +SAVE_RESULT_STATES = (states.SUCCESS, states.FAILURE) + + +@six.add_metaclass(abc.ABCMeta) +class Action(object): + """An action that handles executing, state changes, ... of atoms.""" + + def __init__(self, storage, notifier, walker_factory): + self._storage = storage + self._notifier = notifier + self._walker_factory = walker_factory + + @abc.abstractmethod + def handles(self, atom): + """Checks if this action handles the provided atom.""" diff --git a/taskflow/engines/action_engine/actions/retry.py b/taskflow/engines/action_engine/actions/retry.py index be933ee2..06a81fd4 100644 --- a/taskflow/engines/action_engine/actions/retry.py +++ b/taskflow/engines/action_engine/actions/retry.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +from taskflow.engines.action_engine.actions import base from taskflow.engines.action_engine import executor as ex from taskflow import logging from taskflow import retry as retry_atom @@ -23,16 +24,12 @@ from taskflow.types import futures LOG = logging.getLogger(__name__) -SAVE_RESULT_STATES = (states.SUCCESS, states.FAILURE) - -class RetryAction(object): +class RetryAction(base.Action): """An action that handles executing, state changes, ... of retry atoms.""" def __init__(self, storage, notifier, walker_factory): - self._storage = storage - self._notifier = notifier - self._walker_factory = walker_factory + super(RetryAction, self).__init__(storage, notifier, walker_factory) self._executor = futures.SynchronousExecutor() @staticmethod @@ -50,10 +47,13 @@ class RetryAction(object): kwargs.update(addons) return kwargs - def change_state(self, retry, state, result=None): + def change_state(self, retry, state, result=base.NO_RESULT): old_state = self._storage.get_atom_state(retry.name) - if state in SAVE_RESULT_STATES: - self._storage.save(retry.name, result, state) + if state in base.SAVE_RESULT_STATES: + save_result = None + if result is not base.NO_RESULT: + save_result = result + self._storage.save(retry.name, save_result, state) elif state == states.REVERTED: self._storage.cleanup_retry_history(retry.name, state) else: @@ -66,9 +66,10 @@ class RetryAction(object): details = { 'retry_name': retry.name, 'retry_uuid': retry_uuid, - 'result': result, 'old_state': old_state, } + if result is not base.NO_RESULT: + details['result'] = result self._notifier.notify(state, details) def execute(self, retry): diff --git a/taskflow/engines/action_engine/actions/task.py b/taskflow/engines/action_engine/actions/task.py index fbdc0a8f..607b26d5 100644 --- a/taskflow/engines/action_engine/actions/task.py +++ b/taskflow/engines/action_engine/actions/task.py @@ -16,6 +16,7 @@ import functools +from taskflow.engines.action_engine.actions import base from taskflow import logging from taskflow import states from taskflow import task as task_atom @@ -23,24 +24,20 @@ from taskflow.types import failure LOG = logging.getLogger(__name__) -SAVE_RESULT_STATES = (states.SUCCESS, states.FAILURE) - -class TaskAction(object): +class TaskAction(base.Action): """An action that handles scheduling, state changes, ... of task atoms.""" - def __init__(self, storage, task_executor, notifier, walker_factory): - self._storage = storage + def __init__(self, storage, notifier, walker_factory, task_executor): + super(TaskAction, self).__init__(storage, notifier, walker_factory) self._task_executor = task_executor - self._notifier = notifier - self._walker_factory = walker_factory @staticmethod def handles(atom): return isinstance(atom, task_atom.BaseTask) def _is_identity_transition(self, old_state, state, task, progress): - if state in SAVE_RESULT_STATES: + if state in base.SAVE_RESULT_STATES: # saving result is never identity transition return False if state != old_state: @@ -56,15 +53,19 @@ class TaskAction(object): return False return True - def change_state(self, task, state, result=None, progress=None): + def change_state(self, task, state, + result=base.NO_RESULT, progress=None): old_state = self._storage.get_atom_state(task.name) if self._is_identity_transition(old_state, state, task, progress): # NOTE(imelnikov): ignore identity transitions in order # to avoid extra write to storage backend and, what's # more important, extra notifications return - if state in SAVE_RESULT_STATES: - self._storage.save(task.name, result, state) + if state in base.SAVE_RESULT_STATES: + save_result = None + if result is not base.NO_RESULT: + save_result = result + self._storage.save(task.name, save_result, state) else: self._storage.set_atom_state(task.name, state) if progress is not None: @@ -73,9 +74,10 @@ class TaskAction(object): details = { 'task_name': task.name, 'task_uuid': task_uuid, - 'result': result, 'old_state': old_state, } + if result is not base.NO_RESULT: + details['result'] = result self._notifier.notify(state, details) if progress is not None: task.update_progress(progress) @@ -138,8 +140,8 @@ class TaskAction(object): progress_callback=progress_callback) return future - def complete_reversion(self, task, rev_result): - if isinstance(rev_result, failure.Failure): + def complete_reversion(self, task, result): + if isinstance(result, failure.Failure): self.change_state(task, states.FAILURE) else: self.change_state(task, states.REVERTED, progress=1.0) diff --git a/taskflow/engines/action_engine/runtime.py b/taskflow/engines/action_engine/runtime.py index 8f9b56b5..169a6415 100644 --- a/taskflow/engines/action_engine/runtime.py +++ b/taskflow/engines/action_engine/runtime.py @@ -71,8 +71,9 @@ class Runtime(object): @misc.cachedproperty def task_action(self): - return ta.TaskAction(self._storage, self._task_executor, - self._atom_notifier, self._fetch_scopes_for) + return ta.TaskAction(self._storage, + self._atom_notifier, self._fetch_scopes_for, + self._task_executor) def _fetch_scopes_for(self, atom): """Fetches a tuple of the visible scopes for the given atom.""" diff --git a/taskflow/listeners/base.py b/taskflow/listeners/base.py index 42dc87e9..1600dd3e 100644 --- a/taskflow/listeners/base.py +++ b/taskflow/listeners/base.py @@ -153,6 +153,7 @@ class Listener(object): def __enter__(self): self.register() + return self def __exit__(self, type, value, tb): try: diff --git a/taskflow/tests/unit/conductor/test_conductor.py b/taskflow/tests/unit/conductor/test_conductor.py index 0fb677ea..b861c12b 100644 --- a/taskflow/tests/unit/conductor/test_conductor.py +++ b/taskflow/tests/unit/conductor/test_conductor.py @@ -44,7 +44,7 @@ def close_many(*closeables): def test_factory(blowup): f = lf.Flow("test") if not blowup: - f.add(test_utils.SaveOrderTask('test1')) + f.add(test_utils.ProgressingTask('test1')) else: f.add(test_utils.FailingTask("test1")) return f diff --git a/taskflow/tests/unit/test_engines.py b/taskflow/tests/unit/test_engines.py index baa5e81f..6865bb82 100644 --- a/taskflow/tests/unit/test_engines.py +++ b/taskflow/tests/unit/test_engines.py @@ -17,7 +17,6 @@ import contextlib import testtools -from testtools import testcase import taskflow.engines from taskflow.engines.action_engine import engine as eng @@ -40,53 +39,43 @@ from taskflow.utils import persistence_utils as p_utils from taskflow.utils import threading_utils as tu -class EngineTaskTest(utils.EngineTestBase): +class EngineTaskTest(object): def test_run_task_as_flow(self): - flow = utils.SaveOrderTask(name='task1') + flow = utils.ProgressingTask(name='task1') engine = self._make_engine(flow) - engine.run() - self.assertEqual(self.values, ['task1']) - - @staticmethod - def _callback(state, values, details): - name = details.get('task_name', '') - values.append('%s %s' % (name, state)) - - @staticmethod - def _flow_callback(state, values, details): - values.append('flow %s' % state) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + engine.run() + expected = ['task1.t RUNNING', 'task1.t SUCCESS(5)'] + self.assertEqual(expected, capturer.values) def test_run_task_with_notifications(self): - flow = utils.SaveOrderTask(name='task1') + flow = utils.ProgressingTask(name='task1') engine = self._make_engine(flow) - utils.register_notifiers(engine, self.values) - engine.run() - self.assertEqual(self.values, - ['flow RUNNING', - 'task1 RUNNING', - 'task1', - 'task1 SUCCESS', - 'flow SUCCESS']) + with utils.CaptureListener(engine) as capturer: + engine.run() + expected = ['task1.f RUNNING', 'task1.t RUNNING', + 'task1.t SUCCESS(5)', 'task1.f SUCCESS'] + self.assertEqual(expected, capturer.values) def test_failing_task_with_notifications(self): + values = [] flow = utils.FailingTask('fail') engine = self._make_engine(flow) - utils.register_notifiers(engine, self.values) - expected = ['flow RUNNING', - 'fail RUNNING', - 'fail FAILURE', - 'fail REVERTING', - 'fail reverted(Failure: RuntimeError: Woot!)', - 'fail REVERTED', - 'flow REVERTED'] - self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) - self.assertEqual(self.values, expected) + expected = ['fail.f RUNNING', 'fail.t RUNNING', + 'fail.t FAILURE(Failure: RuntimeError: Woot!)', + 'fail.t REVERTING', 'fail.t REVERTED', + 'fail.f REVERTED'] + with utils.CaptureListener(engine, values=values) as capturer: + self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) + self.assertEqual(expected, capturer.values) self.assertEqual(engine.storage.get_flow_state(), states.REVERTED) - - self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) - now_expected = expected + ['fail PENDING', 'flow PENDING'] + expected - self.assertEqual(self.values, now_expected) + with utils.CaptureListener(engine, values=values) as capturer: + self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) + now_expected = list(expected) + now_expected.extend(['fail.t PENDING', 'fail.f PENDING']) + now_expected.extend(expected) + self.assertEqual(now_expected, values) self.assertEqual(engine.storage.get_flow_state(), states.REVERTED) def test_invalid_flow_raises(self): @@ -124,63 +113,74 @@ class EngineLinearFlowTest(utils.EngineTestBase): def test_sequential_flow_one_task(self): flow = lf.Flow('flow-1').add( - utils.SaveOrderTask(name='task1') + utils.ProgressingTask(name='task1') ) - self._make_engine(flow).run() - self.assertEqual(self.values, ['task1']) + engine = self._make_engine(flow) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + engine.run() + expected = ['task1.t RUNNING', 'task1.t SUCCESS(5)'] + self.assertEqual(expected, capturer.values) def test_sequential_flow_two_tasks(self): flow = lf.Flow('flow-2').add( - utils.SaveOrderTask(name='task1'), - utils.SaveOrderTask(name='task2') + utils.ProgressingTask(name='task1'), + utils.ProgressingTask(name='task2') ) - self._make_engine(flow).run() - self.assertEqual(self.values, ['task1', 'task2']) + engine = self._make_engine(flow) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + engine.run() + expected = ['task1.t RUNNING', 'task1.t SUCCESS(5)', + 'task2.t RUNNING', 'task2.t SUCCESS(5)'] + self.assertEqual(expected, capturer.values) self.assertEqual(len(flow), 2) def test_sequential_flow_two_tasks_iter(self): flow = lf.Flow('flow-2').add( - utils.SaveOrderTask(name='task1'), - utils.SaveOrderTask(name='task2') + utils.ProgressingTask(name='task1'), + utils.ProgressingTask(name='task2') ) - e = self._make_engine(flow) - gathered_states = list(e.run_iter()) + engine = self._make_engine(flow) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + gathered_states = list(engine.run_iter()) self.assertTrue(len(gathered_states) > 0) - self.assertEqual(self.values, ['task1', 'task2']) + expected = ['task1.t RUNNING', 'task1.t SUCCESS(5)', + 'task2.t RUNNING', 'task2.t SUCCESS(5)'] + self.assertEqual(expected, capturer.values) self.assertEqual(len(flow), 2) def test_sequential_flow_iter_suspend_resume(self): flow = lf.Flow('flow-2').add( - utils.SaveOrderTask(name='task1'), - utils.SaveOrderTask(name='task2') + utils.ProgressingTask(name='task1'), + utils.ProgressingTask(name='task2') ) - _lb, fd = p_utils.temporary_flow_detail(self.backend) - e = self._make_engine(flow, flow_detail=fd) - it = e.run_iter() - gathered_states = [] - suspend_it = None - while True: - try: - s = it.send(suspend_it) - gathered_states.append(s) - if s == states.WAITING: - # Stop it before task2 runs/starts. - suspend_it = True - except StopIteration: - break + lb, fd = p_utils.temporary_flow_detail(self.backend) + + engine = self._make_engine(flow, flow_detail=fd) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + it = engine.run_iter() + gathered_states = [] + suspend_it = None + while True: + try: + s = it.send(suspend_it) + gathered_states.append(s) + if s == states.WAITING: + # Stop it before task2 runs/starts. + suspend_it = True + except StopIteration: + break self.assertTrue(len(gathered_states) > 0) - self.assertEqual(self.values, ['task1']) - self.assertEqual(states.SUSPENDED, e.storage.get_flow_state()) + expected = ['task1.t RUNNING', 'task1.t SUCCESS(5)'] + self.assertEqual(expected, capturer.values) + self.assertEqual(states.SUSPENDED, engine.storage.get_flow_state()) # Attempt to resume it and see what runs now... - # - # NOTE(harlowja): Clear all the values, but don't reset the reference. - while len(self.values): - self.values.pop() - gathered_states = list(e.run_iter()) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + gathered_states = list(engine.run_iter()) self.assertTrue(len(gathered_states) > 0) - self.assertEqual(self.values, ['task2']) - self.assertEqual(states.SUCCESS, e.storage.get_flow_state()) + expected = ['task2.t RUNNING', 'task2.t SUCCESS(5)'] + self.assertEqual(expected, capturer.values) + self.assertEqual(states.SUCCESS, engine.storage.get_flow_state()) def test_revert_removes_data(self): flow = lf.Flow('revert-removes').add( @@ -194,13 +194,17 @@ class EngineLinearFlowTest(utils.EngineTestBase): def test_sequential_flow_nested_blocks(self): flow = lf.Flow('nested-1').add( - utils.SaveOrderTask('task1'), + utils.ProgressingTask('task1'), lf.Flow('inner-1').add( - utils.SaveOrderTask('task2') + utils.ProgressingTask('task2') ) ) - self._make_engine(flow).run() - self.assertEqual(self.values, ['task1', 'task2']) + engine = self._make_engine(flow) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + engine.run() + expected = ['task1.t RUNNING', 'task1.t SUCCESS(5)', + 'task2.t RUNNING', 'task2.t SUCCESS(5)'] + self.assertEqual(expected, capturer.values) def test_revert_exception_is_reraised(self): flow = lf.Flow('revert-1').add( @@ -216,26 +220,32 @@ class EngineLinearFlowTest(utils.EngineTestBase): utils.NeverRunningTask(), ) engine = self._make_engine(flow) - self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) - self.assertEqual( - self.values, - ['fail reverted(Failure: RuntimeError: Woot!)']) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) + expected = ['fail.t RUNNING', + 'fail.t FAILURE(Failure: RuntimeError: Woot!)', + 'fail.t REVERTING', 'fail.t REVERTED'] + self.assertEqual(expected, capturer.values) def test_correctly_reverts_children(self): flow = lf.Flow('root-1').add( - utils.SaveOrderTask('task1'), + utils.ProgressingTask('task1'), lf.Flow('child-1').add( - utils.SaveOrderTask('task2'), + utils.ProgressingTask('task2'), utils.FailingTask('fail') ) ) engine = self._make_engine(flow) - self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) - self.assertEqual( - self.values, - ['task1', 'task2', - 'fail reverted(Failure: RuntimeError: Woot!)', - 'task2 reverted(5)', 'task1 reverted(5)']) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) + expected = ['task1.t RUNNING', 'task1.t SUCCESS(5)', + 'task2.t RUNNING', 'task2.t SUCCESS(5)', + 'fail.t RUNNING', + 'fail.t FAILURE(Failure: RuntimeError: Woot!)', + 'fail.t REVERTING', 'fail.t REVERTED', + 'task2.t REVERTING', 'task2.t REVERTED', + 'task1.t REVERTING', 'task1.t REVERTED'] + self.assertEqual(expected, capturer.values) class EngineParallelFlowTest(utils.EngineTestBase): @@ -247,22 +257,26 @@ class EngineParallelFlowTest(utils.EngineTestBase): def test_parallel_flow_one_task(self): flow = uf.Flow('p-1').add( - utils.SaveOrderTask(name='task1', provides='a') + utils.ProgressingTask(name='task1', provides='a') ) engine = self._make_engine(flow) - engine.run() - self.assertEqual(self.values, ['task1']) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + engine.run() + expected = ['task1.t RUNNING', 'task1.t SUCCESS(5)'] + self.assertEqual(expected, capturer.values) self.assertEqual(engine.storage.fetch_all(), {'a': 5}) def test_parallel_flow_two_tasks(self): flow = uf.Flow('p-2').add( - utils.SaveOrderTask(name='task1'), - utils.SaveOrderTask(name='task2') + utils.ProgressingTask(name='task1'), + utils.ProgressingTask(name='task2') ) - self._make_engine(flow).run() - - result = set(self.values) - self.assertEqual(result, set(['task1', 'task2'])) + engine = self._make_engine(flow) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + engine.run() + expected = set(['task2.t SUCCESS(5)', 'task2.t RUNNING', + 'task1.t RUNNING', 'task1.t SUCCESS(5)']) + self.assertEqual(expected, set(capturer.values)) def test_parallel_revert(self): flow = uf.Flow('p-r-3').add( @@ -271,9 +285,10 @@ class EngineParallelFlowTest(utils.EngineTestBase): utils.TaskNoRequiresNoReturns(name='task2') ) engine = self._make_engine(flow) - self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) - self.assertIn('fail reverted(Failure: RuntimeError: Woot!)', - self.values) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) + self.assertIn('fail.t FAILURE(Failure: RuntimeError: Woot!)', + capturer.values) def test_parallel_revert_exception_is_reraised(self): # NOTE(imelnikov): if we put NastyTask and FailingTask @@ -292,12 +307,12 @@ class EngineParallelFlowTest(utils.EngineTestBase): def test_sequential_flow_two_tasks_with_resumption(self): flow = lf.Flow('lf-2-r').add( - utils.SaveOrderTask(name='task1', provides='x1'), - utils.SaveOrderTask(name='task2', provides='x2') + utils.ProgressingTask(name='task1', provides='x1'), + utils.ProgressingTask(name='task2', provides='x2') ) # Create FlowDetail as if we already run task1 - _lb, fd = p_utils.temporary_flow_detail(self.backend) + lb, fd = p_utils.temporary_flow_detail(self.backend) td = logbook.TaskDetail(name='task1', uuid='42') td.state = states.SUCCESS td.results = 17 @@ -308,8 +323,10 @@ class EngineParallelFlowTest(utils.EngineTestBase): td.update(conn.update_atom_details(td)) engine = self._make_engine(flow, fd) - engine.run() - self.assertEqual(self.values, ['task2']) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + engine.run() + expected = ['task2.t RUNNING', 'task2.t SUCCESS(5)'] + self.assertEqual(expected, capturer.values) self.assertEqual(engine.storage.fetch_all(), {'x1': 17, 'x2': 5}) @@ -318,86 +335,98 @@ class EngineLinearAndUnorderedExceptionsTest(utils.EngineTestBase): def test_revert_ok_for_unordered_in_linear(self): flow = lf.Flow('p-root').add( - utils.SaveOrderTask(name='task1'), - utils.SaveOrderTask(name='task2'), + utils.ProgressingTask(name='task1'), + utils.ProgressingTask(name='task2'), uf.Flow('p-inner').add( - utils.SaveOrderTask(name='task3'), + utils.ProgressingTask(name='task3'), utils.FailingTask('fail') ) ) engine = self._make_engine(flow) - self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) # NOTE(imelnikov): we don't know if task 3 was run, but if it was, # it should have been reverted in correct order. possible_values_no_task3 = [ - 'task1', 'task2', - 'fail reverted(Failure: RuntimeError: Woot!)', - 'task2 reverted(5)', 'task1 reverted(5)' + 'task1.t RUNNING', 'task2.t RUNNING', + 'fail.t FAILURE(Failure: RuntimeError: Woot!)', + 'task2.t REVERTED', 'task1.t REVERTED' ] - self.assertIsSuperAndSubsequence(self.values, + self.assertIsSuperAndSubsequence(capturer.values, possible_values_no_task3) - if 'task3' in self.values: + if 'task3' in capturer.values: possible_values_task3 = [ - 'task1', 'task2', 'task3', - 'task3 reverted(5)', 'task2 reverted(5)', 'task1 reverted(5)' + 'task1.t RUNNING', 'task2.t RUNNING', 'task3.t RUNNING', + 'task3.t REVERTED', 'task2.t REVERTED', 'task1.t REVERTED' ] - self.assertIsSuperAndSubsequence(self.values, + self.assertIsSuperAndSubsequence(capturer.values, possible_values_task3) def test_revert_raises_for_unordered_in_linear(self): flow = lf.Flow('p-root').add( - utils.SaveOrderTask(name='task1'), - utils.SaveOrderTask(name='task2'), + utils.ProgressingTask(name='task1'), + utils.ProgressingTask(name='task2'), uf.Flow('p-inner').add( - utils.SaveOrderTask(name='task3'), - utils.NastyFailingTask() + utils.ProgressingTask(name='task3'), + utils.NastyFailingTask(name='nasty') ) ) engine = self._make_engine(flow) - self.assertFailuresRegexp(RuntimeError, '^Gotcha', engine.run) + with utils.CaptureListener(engine, + capture_flow=False, + skip_tasks=['nasty']) as capturer: + self.assertFailuresRegexp(RuntimeError, '^Gotcha', engine.run) # NOTE(imelnikov): we don't know if task 3 was run, but if it was, # it should have been reverted in correct order. - possible_values = ['task1', 'task2', 'task3', - 'task3 reverted(5)'] - self.assertIsSuperAndSubsequence(possible_values, self.values) - possible_values_no_task3 = ['task1', 'task2'] - self.assertIsSuperAndSubsequence(self.values, + possible_values = ['task1.t RUNNING', 'task1.t SUCCESS(5)', + 'task2.t RUNNING', 'task2.t SUCCESS(5)', + 'task3.t RUNNING', 'task3.t SUCCESS(5)', + 'task3.t REVERTING', + 'task3.t REVERTED'] + self.assertIsSuperAndSubsequence(possible_values, capturer.values) + possible_values_no_task3 = ['task1.t RUNNING', 'task2.t RUNNING'] + self.assertIsSuperAndSubsequence(capturer.values, possible_values_no_task3) def test_revert_ok_for_linear_in_unordered(self): flow = uf.Flow('p-root').add( - utils.SaveOrderTask(name='task1'), + utils.ProgressingTask(name='task1'), lf.Flow('p-inner').add( - utils.SaveOrderTask(name='task2'), + utils.ProgressingTask(name='task2'), utils.FailingTask('fail') ) ) engine = self._make_engine(flow) - self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) - self.assertIn('fail reverted(Failure: RuntimeError: Woot!)', - self.values) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) + self.assertIn('fail.t FAILURE(Failure: RuntimeError: Woot!)', + capturer.values) # NOTE(imelnikov): if task1 was run, it should have been reverted. - if 'task1' in self.values: - task1_story = ['task1', 'task1 reverted(5)'] - self.assertIsSuperAndSubsequence(self.values, task1_story) + if 'task1' in capturer.values: + task1_story = ['task1.t RUNNING', 'task1.t SUCCESS(5)', + 'task1.t REVERTED'] + self.assertIsSuperAndSubsequence(capturer.values, task1_story) + # NOTE(imelnikov): task2 should have been run and reverted - task2_story = ['task2', 'task2 reverted(5)'] - self.assertIsSuperAndSubsequence(self.values, task2_story) + task2_story = ['task2.t RUNNING', 'task2.t SUCCESS(5)', + 'task2.t REVERTED'] + self.assertIsSuperAndSubsequence(capturer.values, task2_story) def test_revert_raises_for_linear_in_unordered(self): flow = uf.Flow('p-root').add( - utils.SaveOrderTask(name='task1'), + utils.ProgressingTask(name='task1'), lf.Flow('p-inner').add( - utils.SaveOrderTask(name='task2'), + utils.ProgressingTask(name='task2'), utils.NastyFailingTask() ) ) engine = self._make_engine(flow) - self.assertFailuresRegexp(RuntimeError, '^Gotcha', engine.run) - self.assertNotIn('task2 reverted(5)', self.values) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + self.assertFailuresRegexp(RuntimeError, '^Gotcha', engine.run) + self.assertNotIn('task2.t REVERTED', capturer.values) class EngineGraphFlowTest(utils.EngineTestBase): @@ -415,66 +444,90 @@ class EngineGraphFlowTest(utils.EngineTestBase): def test_graph_flow_one_task(self): flow = gf.Flow('g-1').add( - utils.SaveOrderTask(name='task1') + utils.ProgressingTask(name='task1') ) - self._make_engine(flow).run() - self.assertEqual(self.values, ['task1']) + engine = self._make_engine(flow) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + engine.run() + expected = ['task1.t RUNNING', 'task1.t SUCCESS(5)'] + self.assertEqual(expected, capturer.values) def test_graph_flow_two_independent_tasks(self): flow = gf.Flow('g-2').add( - utils.SaveOrderTask(name='task1'), - utils.SaveOrderTask(name='task2') + utils.ProgressingTask(name='task1'), + utils.ProgressingTask(name='task2') ) - self._make_engine(flow).run() - self.assertEqual(set(self.values), set(['task1', 'task2'])) + engine = self._make_engine(flow) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + engine.run() + expected = set(['task2.t SUCCESS(5)', 'task2.t RUNNING', + 'task1.t RUNNING', 'task1.t SUCCESS(5)']) + self.assertEqual(expected, set(capturer.values)) self.assertEqual(len(flow), 2) def test_graph_flow_two_tasks(self): flow = gf.Flow('g-1-1').add( - utils.SaveOrderTask(name='task2', requires=['a']), - utils.SaveOrderTask(name='task1', provides='a') + utils.ProgressingTask(name='task2', requires=['a']), + utils.ProgressingTask(name='task1', provides='a') ) - self._make_engine(flow).run() - self.assertEqual(self.values, ['task1', 'task2']) + engine = self._make_engine(flow) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + engine.run() + expected = ['task1.t RUNNING', 'task1.t SUCCESS(5)', + 'task2.t RUNNING', 'task2.t SUCCESS(5)'] + self.assertEqual(expected, capturer.values) def test_graph_flow_four_tasks_added_separately(self): flow = (gf.Flow('g-4') - .add(utils.SaveOrderTask(name='task4', - provides='d', requires=['c'])) - .add(utils.SaveOrderTask(name='task2', - provides='b', requires=['a'])) - .add(utils.SaveOrderTask(name='task3', - provides='c', requires=['b'])) - .add(utils.SaveOrderTask(name='task1', - provides='a')) + .add(utils.ProgressingTask(name='task4', + provides='d', requires=['c'])) + .add(utils.ProgressingTask(name='task2', + provides='b', requires=['a'])) + .add(utils.ProgressingTask(name='task3', + provides='c', requires=['b'])) + .add(utils.ProgressingTask(name='task1', + provides='a')) ) - self._make_engine(flow).run() - self.assertEqual(self.values, ['task1', 'task2', 'task3', 'task4']) + engine = self._make_engine(flow) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + engine.run() + expected = ['task1.t RUNNING', 'task1.t SUCCESS(5)', + 'task2.t RUNNING', 'task2.t SUCCESS(5)', + 'task3.t RUNNING', 'task3.t SUCCESS(5)', + 'task4.t RUNNING', 'task4.t SUCCESS(5)'] + self.assertEqual(expected, capturer.values) def test_graph_flow_four_tasks_revert(self): flow = gf.Flow('g-4-failing').add( - utils.SaveOrderTask(name='task4', - provides='d', requires=['c']), - utils.SaveOrderTask(name='task2', - provides='b', requires=['a']), + utils.ProgressingTask(name='task4', + provides='d', requires=['c']), + utils.ProgressingTask(name='task2', + provides='b', requires=['a']), utils.FailingTask(name='task3', provides='c', requires=['b']), - utils.SaveOrderTask(name='task1', provides='a')) + utils.ProgressingTask(name='task1', provides='a')) engine = self._make_engine(flow) - self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) - self.assertEqual( - self.values, - ['task1', 'task2', - 'task3 reverted(Failure: RuntimeError: Woot!)', - 'task2 reverted(5)', 'task1 reverted(5)']) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run) + expected = ['task1.t RUNNING', 'task1.t SUCCESS(5)', + 'task2.t RUNNING', 'task2.t SUCCESS(5)', + 'task3.t RUNNING', + 'task3.t FAILURE(Failure: RuntimeError: Woot!)', + 'task3.t REVERTING', + 'task3.t REVERTED', + 'task2.t REVERTING', + 'task2.t REVERTED', + 'task1.t REVERTING', + 'task1.t REVERTED'] + self.assertEqual(expected, capturer.values) self.assertEqual(engine.storage.get_flow_state(), states.REVERTED) def test_graph_flow_four_tasks_revert_failure(self): flow = gf.Flow('g-3-nasty').add( utils.NastyTask(name='task2', provides='b', requires=['a']), utils.FailingTask(name='task3', requires=['b']), - utils.SaveOrderTask(name='task1', provides='a')) + utils.ProgressingTask(name='task1', provides='a')) engine = self._make_engine(flow) self.assertFailuresRegexp(RuntimeError, '^Gotcha', engine.run) @@ -520,6 +573,9 @@ class EngineGraphFlowTest(utils.EngineTestBase): class EngineCheckingTaskTest(utils.EngineTestBase): + # FIXME: this test uses a inner class that workers/process engines can't + # get to, so we need to do something better to make this test useful for + # those engines... def test_flow_failures_are_passed_to_revert(self): class CheckingTask(task.Task): @@ -541,13 +597,13 @@ class EngineCheckingTaskTest(utils.EngineTestBase): self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) -class SingleThreadedEngineTest(EngineTaskTest, - EngineLinearFlowTest, - EngineParallelFlowTest, - EngineLinearAndUnorderedExceptionsTest, - EngineGraphFlowTest, - EngineCheckingTaskTest, - test.TestCase): +class SerialEngineTest(EngineTaskTest, + EngineLinearFlowTest, + EngineParallelFlowTest, + EngineLinearAndUnorderedExceptionsTest, + EngineGraphFlowTest, + EngineCheckingTaskTest, + test.TestCase): def _make_engine(self, flow, flow_detail=None): return taskflow.engines.load(flow, flow_detail=flow_detail, @@ -563,18 +619,23 @@ class SingleThreadedEngineTest(EngineTaskTest, self.assertIsInstance(engine, eng.SerialActionEngine) -class MultiThreadedEngineTest(EngineTaskTest, - EngineLinearFlowTest, - EngineParallelFlowTest, - EngineLinearAndUnorderedExceptionsTest, - EngineGraphFlowTest, - EngineCheckingTaskTest, - test.TestCase): +class ParallelEngineWithThreadsTest(EngineTaskTest, + EngineLinearFlowTest, + EngineParallelFlowTest, + EngineLinearAndUnorderedExceptionsTest, + EngineGraphFlowTest, + EngineCheckingTaskTest, + test.TestCase): + _EXECUTOR_WORKERS = 2 + def _make_engine(self, flow, flow_detail=None, executor=None): + if executor is None: + executor = 'threads' return taskflow.engines.load(flow, flow_detail=flow_detail, backend=self.backend, executor=executor, - engine='parallel') + engine='parallel', + max_workers=self._EXECUTOR_WORKERS) def test_correct_load(self): engine = self._make_engine(utils.TaskNoRequiresNoReturns) @@ -582,7 +643,7 @@ class MultiThreadedEngineTest(EngineTaskTest, def test_using_common_executor(self): flow = utils.TaskNoRequiresNoReturns(name='task1') - executor = futures.ThreadPoolExecutor(2) + executor = futures.ThreadPoolExecutor(self._EXECUTOR_WORKERS) try: e1 = self._make_engine(flow, executor=executor) e2 = self._make_engine(flow, executor=executor) @@ -614,9 +675,8 @@ class ParallelEngineWithProcessTest(EngineTaskTest, EngineParallelFlowTest, EngineLinearAndUnorderedExceptionsTest, EngineGraphFlowTest, - EngineCheckingTaskTest, test.TestCase): - _SKIP_TYPES = (utils.SaveOrderTask,) + _EXECUTOR_WORKERS = 2 def test_correct_load(self): engine = self._make_engine(utils.TaskNoRequiresNoReturns) @@ -624,27 +684,12 @@ class ParallelEngineWithProcessTest(EngineTaskTest, def _make_engine(self, flow, flow_detail=None, executor=None): if executor is None: - executor = futures.ProcessPoolExecutor(1) - self.addCleanup(executor.shutdown) - e = taskflow.engines.load(flow, flow_detail=flow_detail, - backend=self.backend, engine='parallel', - executor=executor) - # FIXME(harlowja): fix this so that we can actually tests these - # testcases, without having task/global test state that is retained - # and inspected; this doesn't work in a multi-process situation since - # the tasks execute in another process with its own memory/heap - # which this process later can't view/introspect... - try: - e.compile() - for a in e.compilation.execution_graph: - if isinstance(a, self._SKIP_TYPES): - baddies = [a.__name__ for a in self._SKIP_TYPES] - raise testcase.TestSkipped("Process engines can not" - " run flows that contain" - " %s tasks" % baddies) - except (TypeError, exc.TaskFlowException): - pass - return e + executor = 'processes' + return taskflow.engines.load(flow, flow_detail=flow_detail, + backend=self.backend, + engine='parallel', + executor=executor, + max_workers=self._EXECUTOR_WORKERS) class WorkerBasedEngineTest(EngineTaskTest, diff --git a/taskflow/tests/unit/test_retries.py b/taskflow/tests/unit/test_retries.py index 1a2e7e26..27a90d26 100644 --- a/taskflow/tests/unit/test_retries.py +++ b/taskflow/tests/unit/test_retries.py @@ -14,6 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. +import testtools + import taskflow.engines from taskflow import exceptions as exc from taskflow.patterns import graph_flow as gf @@ -24,6 +26,25 @@ from taskflow import states as st from taskflow import test from taskflow.tests import utils from taskflow.types import failure +from taskflow.types import futures +from taskflow.utils import async_utils as au + + +class FailingRetry(retry.Retry): + + def execute(self, **kwargs): + raise ValueError('OMG I FAILED') + + def revert(self, history, **kwargs): + self.history = history + + def on_failure(self, **kwargs): + return retry.REVERT + + +class NastyFailingRetry(FailingRetry): + def revert(self, history, **kwargs): + raise ValueError('WOOT!') class RetryTest(utils.EngineTestBase): @@ -48,89 +69,71 @@ class RetryTest(utils.EngineTestBase): def test_states_retry_success_linear_flow(self): flow = lf.Flow('flow-1', retry.Times(4, 'r1', provides='x')).add( - utils.SaveOrderTask("task1"), + utils.ProgressingTask("task1"), utils.ConditionalTask("task2") ) engine = self._make_engine(flow) - utils.register_notifiers(engine, self.values) engine.storage.inject({'y': 2}) - engine.run() + with utils.CaptureListener(engine) as capturer: + engine.run() self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2}) - expected = ['flow RUNNING', - 'r1 RUNNING', - 'r1 SUCCESS', - 'task1 RUNNING', - 'task1', - 'task1 SUCCESS', - 'task2 RUNNING', - 'task2', - 'task2 FAILURE', - 'task2 REVERTING', - u'task2 reverted(Failure: RuntimeError: Woot!)', - 'task2 REVERTED', - 'task1 REVERTING', - 'task1 reverted(5)', - 'task1 REVERTED', - 'r1 RETRYING', - 'task1 PENDING', - 'task2 PENDING', - 'r1 RUNNING', - 'r1 SUCCESS', - 'task1 RUNNING', - 'task1', - 'task1 SUCCESS', - 'task2 RUNNING', - 'task2', - 'task2 SUCCESS', - 'flow SUCCESS'] - self.assertEqual(self.values, expected) + expected = ['flow-1.f RUNNING', + 'r1.r RUNNING', 'r1.r SUCCESS(1)', + 'task1.t RUNNING', 'task1.t SUCCESS(5)', + 'task2.t RUNNING', + 'task2.t FAILURE(Failure: RuntimeError: Woot!)', + 'task2.t REVERTING', 'task2.t REVERTED', + 'task1.t REVERTING', 'task1.t REVERTED', + 'r1.r RETRYING', + 'task1.t PENDING', + 'task2.t PENDING', + 'r1.r RUNNING', + 'r1.r SUCCESS(2)', + 'task1.t RUNNING', + 'task1.t SUCCESS(5)', + 'task2.t RUNNING', + 'task2.t SUCCESS(None)', + 'flow-1.f SUCCESS'] + self.assertEqual(expected, capturer.values) def test_states_retry_reverted_linear_flow(self): flow = lf.Flow('flow-1', retry.Times(2, 'r1', provides='x')).add( - utils.SaveOrderTask("task1"), + utils.ProgressingTask("task1"), utils.ConditionalTask("task2") ) engine = self._make_engine(flow) - utils.register_notifiers(engine, self.values) engine.storage.inject({'y': 4}) - self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) + with utils.CaptureListener(engine) as capturer: + self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) self.assertEqual(engine.storage.fetch_all(), {'y': 4}) - expected = ['flow RUNNING', - 'r1 RUNNING', - 'r1 SUCCESS', - 'task1 RUNNING', - 'task1', - 'task1 SUCCESS', - 'task2 RUNNING', - 'task2', - 'task2 FAILURE', - 'task2 REVERTING', - u'task2 reverted(Failure: RuntimeError: Woot!)', - 'task2 REVERTED', - 'task1 REVERTING', - 'task1 reverted(5)', - 'task1 REVERTED', - 'r1 RETRYING', - 'task1 PENDING', - 'task2 PENDING', - 'r1 RUNNING', - 'r1 SUCCESS', - 'task1 RUNNING', - 'task1', - 'task1 SUCCESS', - 'task2 RUNNING', - 'task2', - 'task2 FAILURE', - 'task2 REVERTING', - u'task2 reverted(Failure: RuntimeError: Woot!)', - 'task2 REVERTED', - 'task1 REVERTING', - 'task1 reverted(5)', - 'task1 REVERTED', - 'r1 REVERTING', - 'r1 REVERTED', - 'flow REVERTED'] - self.assertEqual(self.values, expected) + expected = ['flow-1.f RUNNING', + 'r1.r RUNNING', + 'r1.r SUCCESS(1)', + 'task1.t RUNNING', + 'task1.t SUCCESS(5)', + 'task2.t RUNNING', + 'task2.t FAILURE(Failure: RuntimeError: Woot!)', + 'task2.t REVERTING', + 'task2.t REVERTED', + 'task1.t REVERTING', + 'task1.t REVERTED', + 'r1.r RETRYING', + 'task1.t PENDING', + 'task2.t PENDING', + 'r1.r RUNNING', + 'r1.r SUCCESS(2)', + 'task1.t RUNNING', + 'task1.t SUCCESS(5)', + 'task2.t RUNNING', + 'task2.t FAILURE(Failure: RuntimeError: Woot!)', + 'task2.t REVERTING', + 'task2.t REVERTED', + 'task1.t REVERTING', + 'task1.t REVERTED', + 'r1.r REVERTING', + 'r1.r REVERTED', + 'flow-1.f REVERTED'] + self.assertEqual(expected, capturer.values) def test_states_retry_failure_linear_flow(self): flow = lf.Flow('flow-1', retry.Times(2, 'r1', provides='x')).add( @@ -138,25 +141,23 @@ class RetryTest(utils.EngineTestBase): utils.ConditionalTask("task2") ) engine = self._make_engine(flow) - utils.register_notifiers(engine, self.values) engine.storage.inject({'y': 4}) - self.assertRaisesRegexp(RuntimeError, '^Gotcha', engine.run) + with utils.CaptureListener(engine) as capturer: + self.assertRaisesRegexp(RuntimeError, '^Gotcha', engine.run) self.assertEqual(engine.storage.fetch_all(), {'y': 4, 'x': 1}) - expected = ['flow RUNNING', - 'r1 RUNNING', - 'r1 SUCCESS', - 'task1 RUNNING', - 'task1 SUCCESS', - 'task2 RUNNING', - 'task2', - 'task2 FAILURE', - 'task2 REVERTING', - u'task2 reverted(Failure: RuntimeError: Woot!)', - 'task2 REVERTED', - 'task1 REVERTING', - 'task1 FAILURE', - 'flow FAILURE'] - self.assertEqual(self.values, expected) + expected = ['flow-1.f RUNNING', + 'r1.r RUNNING', + 'r1.r SUCCESS(1)', + 'task1.t RUNNING', + 'task1.t SUCCESS(None)', + 'task2.t RUNNING', + 'task2.t FAILURE(Failure: RuntimeError: Woot!)', + 'task2.t REVERTING', + 'task2.t REVERTED', + 'task1.t REVERTING', + 'task1.t FAILURE', + 'flow-1.f FAILURE'] + self.assertEqual(expected, capturer.values) def test_states_retry_failure_nested_flow_fails(self): flow = lf.Flow('flow-1', utils.retry.AlwaysRevert('r1')).add( @@ -168,41 +169,38 @@ class RetryTest(utils.EngineTestBase): utils.TaskNoRequiresNoReturns("task4") ) engine = self._make_engine(flow) - utils.register_notifiers(engine, self.values) engine.storage.inject({'y': 2}) - engine.run() + with utils.CaptureListener(engine) as capturer: + engine.run() self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2}) - expected = ['flow RUNNING', - 'r1 RUNNING', - 'r1 SUCCESS', - 'task1 RUNNING', - 'task1 SUCCESS', - 'r2 RUNNING', - 'r2 SUCCESS', - 'task2 RUNNING', - 'task2 SUCCESS', - 'task3 RUNNING', - 'task3', - 'task3 FAILURE', - 'task3 REVERTING', - u'task3 reverted(Failure: RuntimeError: Woot!)', - 'task3 REVERTED', - 'task2 REVERTING', - 'task2 REVERTED', - 'r2 RETRYING', - 'task2 PENDING', - 'task3 PENDING', - 'r2 RUNNING', - 'r2 SUCCESS', - 'task2 RUNNING', - 'task2 SUCCESS', - 'task3 RUNNING', - 'task3', - 'task3 SUCCESS', - 'task4 RUNNING', - 'task4 SUCCESS', - 'flow SUCCESS'] - self.assertEqual(self.values, expected) + expected = ['flow-1.f RUNNING', + 'r1.r RUNNING', + 'r1.r SUCCESS(None)', + 'task1.t RUNNING', + 'task1.t SUCCESS(None)', + 'r2.r RUNNING', + 'r2.r SUCCESS(1)', + 'task2.t RUNNING', + 'task2.t SUCCESS(None)', + 'task3.t RUNNING', + 'task3.t FAILURE(Failure: RuntimeError: Woot!)', + 'task3.t REVERTING', + 'task3.t REVERTED', + 'task2.t REVERTING', + 'task2.t REVERTED', + 'r2.r RETRYING', + 'task2.t PENDING', + 'task3.t PENDING', + 'r2.r RUNNING', + 'r2.r SUCCESS(2)', + 'task2.t RUNNING', + 'task2.t SUCCESS(None)', + 'task3.t RUNNING', + 'task3.t SUCCESS(None)', + 'task4.t RUNNING', + 'task4.t SUCCESS(None)', + 'flow-1.f SUCCESS'] + self.assertEqual(expected, capturer.values) def test_states_retry_failure_parent_flow_fails(self): flow = lf.Flow('flow-1', retry.Times(3, 'r1', provides='x1')).add( @@ -214,158 +212,160 @@ class RetryTest(utils.EngineTestBase): utils.ConditionalTask("task4", rebind={'x': 'x1'}) ) engine = self._make_engine(flow) - utils.register_notifiers(engine, self.values) engine.storage.inject({'y': 2}) - engine.run() + with utils.CaptureListener(engine) as capturer: + engine.run() self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x1': 2, 'x2': 1}) - expected = ['flow RUNNING', - 'r1 RUNNING', - 'r1 SUCCESS', - 'task1 RUNNING', - 'task1 SUCCESS', - 'r2 RUNNING', - 'r2 SUCCESS', - 'task2 RUNNING', - 'task2 SUCCESS', - 'task3 RUNNING', - 'task3 SUCCESS', - 'task4 RUNNING', - 'task4', - 'task4 FAILURE', - 'task4 REVERTING', - u'task4 reverted(Failure: RuntimeError: Woot!)', - 'task4 REVERTED', - 'task3 REVERTING', - 'task3 REVERTED', - 'task2 REVERTING', - 'task2 REVERTED', - 'r2 REVERTING', - 'r2 REVERTED', - 'task1 REVERTING', - 'task1 REVERTED', - 'r1 RETRYING', - 'task1 PENDING', - 'r2 PENDING', - 'task2 PENDING', - 'task3 PENDING', - 'task4 PENDING', - 'r1 RUNNING', - 'r1 SUCCESS', - 'task1 RUNNING', - 'task1 SUCCESS', - 'r2 RUNNING', - 'r2 SUCCESS', - 'task2 RUNNING', - 'task2 SUCCESS', - 'task3 RUNNING', - 'task3 SUCCESS', - 'task4 RUNNING', - 'task4', - 'task4 SUCCESS', - 'flow SUCCESS'] - self.assertEqual(self.values, expected) + expected = ['flow-1.f RUNNING', + 'r1.r RUNNING', + 'r1.r SUCCESS(1)', + 'task1.t RUNNING', + 'task1.t SUCCESS(None)', + 'r2.r RUNNING', + 'r2.r SUCCESS(1)', + 'task2.t RUNNING', + 'task2.t SUCCESS(None)', + 'task3.t RUNNING', + 'task3.t SUCCESS(None)', + 'task4.t RUNNING', + 'task4.t FAILURE(Failure: RuntimeError: Woot!)', + 'task4.t REVERTING', + 'task4.t REVERTED', + 'task3.t REVERTING', + 'task3.t REVERTED', + 'task2.t REVERTING', + 'task2.t REVERTED', + 'r2.r REVERTING', + 'r2.r REVERTED', + 'task1.t REVERTING', + 'task1.t REVERTED', + 'r1.r RETRYING', + 'task1.t PENDING', + 'r2.r PENDING', + 'task2.t PENDING', + 'task3.t PENDING', + 'task4.t PENDING', + 'r1.r RUNNING', + 'r1.r SUCCESS(2)', + 'task1.t RUNNING', + 'task1.t SUCCESS(None)', + 'r2.r RUNNING', + 'r2.r SUCCESS(1)', + 'task2.t RUNNING', + 'task2.t SUCCESS(None)', + 'task3.t RUNNING', + 'task3.t SUCCESS(None)', + 'task4.t RUNNING', + 'task4.t SUCCESS(None)', + 'flow-1.f SUCCESS'] + self.assertEqual(expected, capturer.values) def test_unordered_flow_task_fails_parallel_tasks_should_be_reverted(self): flow = uf.Flow('flow-1', retry.Times(3, 'r', provides='x')).add( - utils.SaveOrderTask("task1"), + utils.ProgressingTask("task1"), utils.ConditionalTask("task2") ) engine = self._make_engine(flow) engine.storage.inject({'y': 2}) - engine.run() + with utils.CaptureListener(engine) as capturer: + engine.run() self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2}) - expected = ['task2', - 'task1', - u'task2 reverted(Failure: RuntimeError: Woot!)', - 'task1 reverted(5)', - 'task2', - 'task1'] - self.assertItemsEqual(self.values, expected) + expected = ['flow-1.f RUNNING', + 'r.r RUNNING', + 'r.r SUCCESS(1)', + 'task1.t RUNNING', + 'task2.t RUNNING', + 'task1.t SUCCESS(5)', + 'task2.t FAILURE(Failure: RuntimeError: Woot!)', + 'task2.t REVERTING', + 'task1.t REVERTING', + 'task2.t REVERTED', + 'task1.t REVERTED', + 'r.r RETRYING', + 'task1.t PENDING', + 'task2.t PENDING', + 'r.r RUNNING', + 'r.r SUCCESS(2)', + 'task1.t RUNNING', + 'task2.t RUNNING', + 'task1.t SUCCESS(5)', + 'task2.t SUCCESS(None)', + 'flow-1.f SUCCESS'] + self.assertItemsEqual(capturer.values, expected) def test_nested_flow_reverts_parent_retries(self): retry1 = retry.Times(3, 'r1', provides='x') retry2 = retry.Times(0, 'r2', provides='x2') - flow = lf.Flow('flow-1', retry1).add( - utils.SaveOrderTask("task1"), + utils.ProgressingTask("task1"), lf.Flow('flow-2', retry2).add(utils.ConditionalTask("task2")) ) engine = self._make_engine(flow) engine.storage.inject({'y': 2}) - utils.register_notifiers(engine, self.values) - engine.run() + with utils.CaptureListener(engine) as capturer: + engine.run() self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2, 'x2': 1}) - expected = ['flow RUNNING', - 'r1 RUNNING', - 'r1 SUCCESS', - 'task1 RUNNING', - 'task1', - 'task1 SUCCESS', - 'r2 RUNNING', - 'r2 SUCCESS', - 'task2 RUNNING', - 'task2', - 'task2 FAILURE', - 'task2 REVERTING', - u'task2 reverted(Failure: RuntimeError: Woot!)', - 'task2 REVERTED', - 'r2 REVERTING', - 'r2 REVERTED', - 'task1 REVERTING', - 'task1 reverted(5)', - 'task1 REVERTED', - 'r1 RETRYING', - 'task1 PENDING', - 'r2 PENDING', - 'task2 PENDING', - 'r1 RUNNING', - 'r1 SUCCESS', - 'task1 RUNNING', - 'task1', - 'task1 SUCCESS', - 'r2 RUNNING', - 'r2 SUCCESS', - 'task2 RUNNING', - 'task2', - 'task2 SUCCESS', - 'flow SUCCESS'] - self.assertEqual(self.values, expected) + expected = ['flow-1.f RUNNING', + 'r1.r RUNNING', + 'r1.r SUCCESS(1)', + 'task1.t RUNNING', + 'task1.t SUCCESS(5)', + 'r2.r RUNNING', + 'r2.r SUCCESS(1)', + 'task2.t RUNNING', + 'task2.t FAILURE(Failure: RuntimeError: Woot!)', + 'task2.t REVERTING', + 'task2.t REVERTED', + 'r2.r REVERTING', + 'r2.r REVERTED', + 'task1.t REVERTING', + 'task1.t REVERTED', + 'r1.r RETRYING', + 'task1.t PENDING', + 'r2.r PENDING', + 'task2.t PENDING', + 'r1.r RUNNING', + 'r1.r SUCCESS(2)', + 'task1.t RUNNING', + 'task1.t SUCCESS(5)', + 'r2.r RUNNING', + 'r2.r SUCCESS(1)', + 'task2.t RUNNING', + 'task2.t SUCCESS(None)', + 'flow-1.f SUCCESS'] + self.assertEqual(expected, capturer.values) def test_revert_all_retry(self): flow = lf.Flow('flow-1', retry.Times(3, 'r1', provides='x')).add( - utils.SaveOrderTask("task1"), + utils.ProgressingTask("task1"), lf.Flow('flow-2', retry.AlwaysRevertAll('r2')).add( utils.ConditionalTask("task2")) ) engine = self._make_engine(flow) engine.storage.inject({'y': 2}) - utils.register_notifiers(engine, self.values) - self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) + with utils.CaptureListener(engine) as capturer: + self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) self.assertEqual(engine.storage.fetch_all(), {'y': 2}) - expected = ['flow RUNNING', - 'r1 RUNNING', - 'r1 SUCCESS', - 'task1 RUNNING', - 'task1', - 'task1 SUCCESS', - 'r2 RUNNING', - 'r2 SUCCESS', - 'task2 RUNNING', - 'task2', - 'task2 FAILURE', - 'task2 REVERTING', - u'task2 reverted(Failure: RuntimeError: Woot!)', - 'task2 REVERTED', - 'r2 REVERTING', - 'r2 REVERTED', - 'task1 REVERTING', - 'task1 reverted(5)', - 'task1 REVERTED', - 'r1 REVERTING', - 'r1 REVERTED', - 'flow REVERTED'] - self.assertEqual(self.values, expected) + expected = ['flow-1.f RUNNING', + 'r1.r RUNNING', + 'r1.r SUCCESS(1)', + 'task1.t RUNNING', + 'task1.t SUCCESS(5)', + 'r2.r RUNNING', + 'r2.r SUCCESS(None)', + 'task2.t RUNNING', + 'task2.t FAILURE(Failure: RuntimeError: Woot!)', + 'task2.t REVERTING', + 'task2.t REVERTED', + 'r2.r REVERTING', + 'r2.r REVERTED', + 'task1.t REVERTING', + 'task1.t REVERTED', + 'r1.r REVERTING', + 'r1.r REVERTED', + 'flow-1.f REVERTED'] + self.assertEqual(expected, capturer.values) def test_restart_reverted_flow_with_retry(self): flow = lf.Flow('test', retry=utils.OneReturnRetry(provides='x')).add( @@ -386,123 +386,213 @@ class RetryTest(utils.EngineTestBase): def test_resume_flow_that_had_been_interrupted_during_retrying(self): flow = lf.Flow('flow-1', retry.Times(3, 'r1')).add( - utils.SaveOrderTask('t1'), - utils.SaveOrderTask('t2'), - utils.SaveOrderTask('t3') + utils.ProgressingTask('t1'), + utils.ProgressingTask('t2'), + utils.ProgressingTask('t3') ) engine = self._make_engine(flow) engine.compile() engine.prepare() - utils.register_notifiers(engine, self.values) - engine.storage.set_atom_state('r1', st.RETRYING) - engine.storage.set_atom_state('t1', st.PENDING) - engine.storage.set_atom_state('t2', st.REVERTED) - engine.storage.set_atom_state('t3', st.REVERTED) - - engine.run() - expected = ['flow RUNNING', - 't2 PENDING', - 't3 PENDING', - 'r1 RUNNING', - 'r1 SUCCESS', - 't1 RUNNING', - 't1', - 't1 SUCCESS', - 't2 RUNNING', - 't2', - 't2 SUCCESS', - 't3 RUNNING', - 't3', - 't3 SUCCESS', - 'flow SUCCESS'] - self.assertEqual(self.values, expected) + with utils.CaptureListener(engine) as capturer: + engine.storage.set_atom_state('r1', st.RETRYING) + engine.storage.set_atom_state('t1', st.PENDING) + engine.storage.set_atom_state('t2', st.REVERTED) + engine.storage.set_atom_state('t3', st.REVERTED) + engine.run() + expected = ['flow-1.f RUNNING', + 't2.t PENDING', + 't3.t PENDING', + 'r1.r RUNNING', + 'r1.r SUCCESS(1)', + 't1.t RUNNING', + 't1.t SUCCESS(5)', + 't2.t RUNNING', + 't2.t SUCCESS(5)', + 't3.t RUNNING', + 't3.t SUCCESS(5)', + 'flow-1.f SUCCESS'] + self.assertEqual(capturer.values, expected) def test_resume_flow_that_should_be_retried(self): flow = lf.Flow('flow-1', retry.Times(3, 'r1')).add( - utils.SaveOrderTask('t1'), - utils.SaveOrderTask('t2') + utils.ProgressingTask('t1'), + utils.ProgressingTask('t2') ) engine = self._make_engine(flow) engine.compile() engine.prepare() - utils.register_notifiers(engine, self.values) - engine.storage.set_atom_intention('r1', st.RETRY) - engine.storage.set_atom_state('r1', st.SUCCESS) - engine.storage.set_atom_state('t1', st.REVERTED) - engine.storage.set_atom_state('t2', st.REVERTED) - - engine.run() - expected = ['flow RUNNING', - 'r1 RETRYING', - 't1 PENDING', - 't2 PENDING', - 'r1 RUNNING', - 'r1 SUCCESS', - 't1 RUNNING', - 't1', - 't1 SUCCESS', - 't2 RUNNING', - 't2', - 't2 SUCCESS', - 'flow SUCCESS'] - self.assertEqual(self.values, expected) + with utils.CaptureListener(engine) as capturer: + engine.storage.set_atom_intention('r1', st.RETRY) + engine.storage.set_atom_state('r1', st.SUCCESS) + engine.storage.set_atom_state('t1', st.REVERTED) + engine.storage.set_atom_state('t2', st.REVERTED) + engine.run() + expected = ['flow-1.f RUNNING', + 'r1.r RETRYING', + 't1.t PENDING', + 't2.t PENDING', + 'r1.r RUNNING', + 'r1.r SUCCESS(1)', + 't1.t RUNNING', + 't1.t SUCCESS(5)', + 't2.t RUNNING', + 't2.t SUCCESS(5)', + 'flow-1.f SUCCESS'] + self.assertEqual(expected, capturer.values) def test_retry_tasks_that_has_not_been_reverted(self): flow = lf.Flow('flow-1', retry.Times(3, 'r1', provides='x')).add( utils.ConditionalTask('c'), - utils.SaveOrderTask('t1') + utils.ProgressingTask('t1') ) engine = self._make_engine(flow) engine.storage.inject({'y': 2}) - engine.run() - expected = ['c', - u'c reverted(Failure: RuntimeError: Woot!)', - 'c', - 't1'] - self.assertEqual(self.values, expected) + with utils.CaptureListener(engine) as capturer: + engine.run() + expected = ['flow-1.f RUNNING', + 'r1.r RUNNING', + 'r1.r SUCCESS(1)', + 'c.t RUNNING', + 'c.t FAILURE(Failure: RuntimeError: Woot!)', + 'c.t REVERTING', + 'c.t REVERTED', + 'r1.r RETRYING', + 'c.t PENDING', + 'r1.r RUNNING', + 'r1.r SUCCESS(2)', + 'c.t RUNNING', + 'c.t SUCCESS(None)', + 't1.t RUNNING', + 't1.t SUCCESS(5)', + 'flow-1.f SUCCESS'] + self.assertEqual(capturer.values, expected) def test_default_times_retry(self): flow = lf.Flow('flow-1', retry.Times(3, 'r1')).add( - utils.SaveOrderTask('t1'), + utils.ProgressingTask('t1'), utils.FailingTask('t2')) engine = self._make_engine(flow) - - self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) - expected = ['t1', - u't2 reverted(Failure: RuntimeError: Woot!)', - 't1 reverted(5)', - 't1', - u't2 reverted(Failure: RuntimeError: Woot!)', - 't1 reverted(5)', - 't1', - u't2 reverted(Failure: RuntimeError: Woot!)', - 't1 reverted(5)'] - self.assertEqual(self.values, expected) + with utils.CaptureListener(engine) as capturer: + self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) + expected = ['flow-1.f RUNNING', + 'r1.r RUNNING', + 'r1.r SUCCESS(1)', + 't1.t RUNNING', + 't1.t SUCCESS(5)', + 't2.t RUNNING', + 't2.t FAILURE(Failure: RuntimeError: Woot!)', + 't2.t REVERTING', + 't2.t REVERTED', + 't1.t REVERTING', + 't1.t REVERTED', + 'r1.r RETRYING', + 't1.t PENDING', + 't2.t PENDING', + 'r1.r RUNNING', + 'r1.r SUCCESS(2)', + 't1.t RUNNING', + 't1.t SUCCESS(5)', + 't2.t RUNNING', + 't2.t FAILURE(Failure: RuntimeError: Woot!)', + 't2.t REVERTING', + 't2.t REVERTED', + 't1.t REVERTING', + 't1.t REVERTED', + 'r1.r RETRYING', + 't1.t PENDING', + 't2.t PENDING', + 'r1.r RUNNING', + 'r1.r SUCCESS(3)', + 't1.t RUNNING', + 't1.t SUCCESS(5)', + 't2.t RUNNING', + 't2.t FAILURE(Failure: RuntimeError: Woot!)', + 't2.t REVERTING', + 't2.t REVERTED', + 't1.t REVERTING', + 't1.t REVERTED', + 'r1.r REVERTING', + 'r1.r REVERTED', + 'flow-1.f REVERTED'] + self.assertEqual(expected, capturer.values) def test_for_each_with_list(self): collection = [3, 2, 3, 5] retry1 = retry.ForEach(collection, 'r1', provides='x') flow = lf.Flow('flow-1', retry1).add(utils.FailingTaskWithOneArg('t1')) engine = self._make_engine(flow) - - self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) - expected = [u't1 reverted(Failure: RuntimeError: Woot with 3)', - u't1 reverted(Failure: RuntimeError: Woot with 2)', - u't1 reverted(Failure: RuntimeError: Woot with 3)', - u't1 reverted(Failure: RuntimeError: Woot with 5)'] - self.assertEqual(self.values, expected) + with utils.CaptureListener(engine) as capturer: + self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) + expected = ['flow-1.f RUNNING', + 'r1.r RUNNING', + 'r1.r SUCCESS(3)', + 't1.t RUNNING', + 't1.t FAILURE(Failure: RuntimeError: Woot with 3)', + 't1.t REVERTING', + 't1.t REVERTED', + 'r1.r RETRYING', + 't1.t PENDING', + 'r1.r RUNNING', + 'r1.r SUCCESS(2)', + 't1.t RUNNING', + 't1.t FAILURE(Failure: RuntimeError: Woot with 2)', + 't1.t REVERTING', + 't1.t REVERTED', + 'r1.r RETRYING', + 't1.t PENDING', + 'r1.r RUNNING', + 'r1.r SUCCESS(3)', + 't1.t RUNNING', + 't1.t FAILURE(Failure: RuntimeError: Woot with 3)', + 't1.t REVERTING', + 't1.t REVERTED', + 'r1.r RETRYING', + 't1.t PENDING', + 'r1.r RUNNING', + 'r1.r SUCCESS(5)', + 't1.t RUNNING', + 't1.t FAILURE(Failure: RuntimeError: Woot with 5)', + 't1.t REVERTING', + 't1.t REVERTED', + 'r1.r REVERTING', + 'r1.r REVERTED', + 'flow-1.f REVERTED'] + self.assertEqual(expected, capturer.values) def test_for_each_with_set(self): collection = set([3, 2, 5]) retry1 = retry.ForEach(collection, 'r1', provides='x') flow = lf.Flow('flow-1', retry1).add(utils.FailingTaskWithOneArg('t1')) engine = self._make_engine(flow) - - self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) - expected = [u't1 reverted(Failure: RuntimeError: Woot with 3)', - u't1 reverted(Failure: RuntimeError: Woot with 2)', - u't1 reverted(Failure: RuntimeError: Woot with 5)'] - self.assertItemsEqual(self.values, expected) + with utils.CaptureListener(engine) as capturer: + self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) + expected = ['flow-1.f RUNNING', + 'r1.r RUNNING', + 'r1.r SUCCESS(2)', + 't1.t RUNNING', + 't1.t FAILURE(Failure: RuntimeError: Woot with 2)', + 't1.t REVERTING', + 't1.t REVERTED', + 'r1.r RETRYING', + 't1.t PENDING', + 'r1.r RUNNING', + 'r1.r SUCCESS(3)', + 't1.t RUNNING', + 't1.t FAILURE(Failure: RuntimeError: Woot with 3)', + 't1.t REVERTING', + 't1.t REVERTED', + 'r1.r RETRYING', + 't1.t PENDING', + 'r1.r RUNNING', + 'r1.r SUCCESS(5)', + 't1.t RUNNING', + 't1.t FAILURE(Failure: RuntimeError: Woot with 5)', + 't1.t REVERTING', + 't1.t REVERTED', + 'r1.r REVERTING', + 'r1.r REVERTED', + 'flow-1.f REVERTED'] + self.assertItemsEqual(capturer.values, expected) def test_for_each_empty_collection(self): values = [] @@ -518,12 +608,35 @@ class RetryTest(utils.EngineTestBase): flow = lf.Flow('flow-1', retry1).add(utils.FailingTaskWithOneArg('t1')) engine = self._make_engine(flow) engine.storage.inject({'values': values, 'y': 1}) - - self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) - expected = [u't1 reverted(Failure: RuntimeError: Woot with 3)', - u't1 reverted(Failure: RuntimeError: Woot with 2)', - u't1 reverted(Failure: RuntimeError: Woot with 5)'] - self.assertEqual(self.values, expected) + with utils.CaptureListener(engine) as capturer: + self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) + expected = ['flow-1.f RUNNING', + 'r1.r RUNNING', + 'r1.r SUCCESS(3)', + 't1.t RUNNING', + 't1.t FAILURE(Failure: RuntimeError: Woot with 3)', + 't1.t REVERTING', + 't1.t REVERTED', + 'r1.r RETRYING', + 't1.t PENDING', + 'r1.r RUNNING', + 'r1.r SUCCESS(2)', + 't1.t RUNNING', + 't1.t FAILURE(Failure: RuntimeError: Woot with 2)', + 't1.t REVERTING', + 't1.t REVERTED', + 'r1.r RETRYING', + 't1.t PENDING', + 'r1.r RUNNING', + 'r1.r SUCCESS(5)', + 't1.t RUNNING', + 't1.t FAILURE(Failure: RuntimeError: Woot with 5)', + 't1.t REVERTING', + 't1.t REVERTED', + 'r1.r REVERTING', + 'r1.r REVERTED', + 'flow-1.f REVERTED'] + self.assertEqual(expected, capturer.values) def test_parameterized_for_each_with_set(self): values = ([3, 2, 5]) @@ -531,12 +644,35 @@ class RetryTest(utils.EngineTestBase): flow = lf.Flow('flow-1', retry1).add(utils.FailingTaskWithOneArg('t1')) engine = self._make_engine(flow) engine.storage.inject({'values': values, 'y': 1}) - - self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) - expected = [u't1 reverted(Failure: RuntimeError: Woot with 3)', - u't1 reverted(Failure: RuntimeError: Woot with 2)', - u't1 reverted(Failure: RuntimeError: Woot with 5)'] - self.assertItemsEqual(self.values, expected) + with utils.CaptureListener(engine) as capturer: + self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) + expected = ['flow-1.f RUNNING', + 'r1.r RUNNING', + 'r1.r SUCCESS(3)', + 't1.t RUNNING', + 't1.t FAILURE(Failure: RuntimeError: Woot with 3)', + 't1.t REVERTING', + 't1.t REVERTED', + 'r1.r RETRYING', + 't1.t PENDING', + 'r1.r RUNNING', + 'r1.r SUCCESS(2)', + 't1.t RUNNING', + 't1.t FAILURE(Failure: RuntimeError: Woot with 2)', + 't1.t REVERTING', + 't1.t REVERTED', + 'r1.r RETRYING', + 't1.t PENDING', + 'r1.r RUNNING', + 'r1.r SUCCESS(5)', + 't1.t RUNNING', + 't1.t FAILURE(Failure: RuntimeError: Woot with 5)', + 't1.t REVERTING', + 't1.t REVERTED', + 'r1.r REVERTING', + 'r1.r REVERTED', + 'flow-1.f REVERTED'] + self.assertItemsEqual(capturer.values, expected) def test_parameterized_for_each_empty_collection(self): values = [] @@ -548,7 +684,7 @@ class RetryTest(utils.EngineTestBase): def _pretend_to_run_a_flow_and_crash(self, when): flow = uf.Flow('flow-1', retry.Times(3, provides='x')).add( - utils.SaveOrderTask('task1')) + utils.ProgressingTask('task1')) engine = self._make_engine(flow) engine.compile() engine.prepare() @@ -583,52 +719,79 @@ class RetryTest(utils.EngineTestBase): def test_resumption_on_crash_after_task_failure(self): engine = self._pretend_to_run_a_flow_and_crash('task fails') - # then process die and we resume engine - engine.run() - expected = [u'task1 reverted(Failure: RuntimeError: foo)', 'task1'] - self.assertEqual(self.values, expected) + with utils.CaptureListener(engine) as capturer: + engine.run() + expected = ['task1.t REVERTING', + 'task1.t REVERTED', + 'flow-1_retry.r RETRYING', + 'task1.t PENDING', + 'flow-1_retry.r RUNNING', + 'flow-1_retry.r SUCCESS(2)', + 'task1.t RUNNING', + 'task1.t SUCCESS(5)', + 'flow-1.f SUCCESS'] + self.assertEqual(expected, capturer.values) def test_resumption_on_crash_after_retry_queried(self): engine = self._pretend_to_run_a_flow_and_crash('retry queried') - # then process die and we resume engine - engine.run() - expected = [u'task1 reverted(Failure: RuntimeError: foo)', 'task1'] - self.assertEqual(self.values, expected) + with utils.CaptureListener(engine) as capturer: + engine.run() + expected = ['task1.t REVERTING', + 'task1.t REVERTED', + 'flow-1_retry.r RETRYING', + 'task1.t PENDING', + 'flow-1_retry.r RUNNING', + 'flow-1_retry.r SUCCESS(2)', + 'task1.t RUNNING', + 'task1.t SUCCESS(5)', + 'flow-1.f SUCCESS'] + self.assertEqual(expected, capturer.values) def test_resumption_on_crash_after_retry_updated(self): engine = self._pretend_to_run_a_flow_and_crash('retry updated') - # then process die and we resume engine - engine.run() - expected = [u'task1 reverted(Failure: RuntimeError: foo)', 'task1'] - self.assertEqual(self.values, expected) + with utils.CaptureListener(engine) as capturer: + engine.run() + expected = ['task1.t REVERTING', + 'task1.t REVERTED', + 'flow-1_retry.r RETRYING', + 'task1.t PENDING', + 'flow-1_retry.r RUNNING', + 'flow-1_retry.r SUCCESS(2)', + 'task1.t RUNNING', + 'task1.t SUCCESS(5)', + 'flow-1.f SUCCESS'] + self.assertEqual(expected, capturer.values) def test_resumption_on_crash_after_task_updated(self): engine = self._pretend_to_run_a_flow_and_crash('task updated') - # then process die and we resume engine - engine.run() - expected = [u'task1 reverted(Failure: RuntimeError: foo)', 'task1'] - self.assertEqual(self.values, expected) + with utils.CaptureListener(engine) as capturer: + engine.run() + expected = ['task1.t REVERTING', + 'task1.t REVERTED', + 'flow-1_retry.r RETRYING', + 'task1.t PENDING', + 'flow-1_retry.r RUNNING', + 'flow-1_retry.r SUCCESS(2)', + 'task1.t RUNNING', + 'task1.t SUCCESS(5)', + 'flow-1.f SUCCESS'] + self.assertEqual(expected, capturer.values) def test_resumption_on_crash_after_revert_scheduled(self): engine = self._pretend_to_run_a_flow_and_crash('revert scheduled') - # then process die and we resume engine - engine.run() - expected = [u'task1 reverted(Failure: RuntimeError: foo)', 'task1'] - self.assertEqual(self.values, expected) + with utils.CaptureListener(engine) as capturer: + engine.run() + expected = ['task1.t REVERTED', + 'flow-1_retry.r RETRYING', + 'task1.t PENDING', + 'flow-1_retry.r RUNNING', + 'flow-1_retry.r SUCCESS(2)', + 'task1.t RUNNING', + 'task1.t SUCCESS(5)', + 'flow-1.f SUCCESS'] + self.assertEqual(capturer.values, expected) def test_retry_fails(self): - - class FailingRetry(retry.Retry): - - def execute(self, **kwargs): - raise ValueError('OMG I FAILED') - - def revert(self, history, **kwargs): - self.history = history - - def on_failure(self, **kwargs): - return retry.REVERT - r = FailingRetry() flow = lf.Flow('testflow', r) engine = self._make_engine(flow) @@ -640,28 +803,16 @@ class RetryTest(utils.EngineTestBase): self.assertTrue(r.history.caused_by(ValueError, include_retry=True)) def test_retry_revert_fails(self): - - class FailingRetry(retry.Retry): - - def execute(self, **kwargs): - raise ValueError('OMG I FAILED') - - def revert(self, history, **kwargs): - raise ValueError('WOOT!') - - def on_failure(self, **kwargs): - return retry.REVERT - - r = FailingRetry() + r = NastyFailingRetry() flow = lf.Flow('testflow', r) engine = self._make_engine(flow) self.assertRaisesRegexp(ValueError, '^WOOT', engine.run) def test_nested_provides_graph_reverts_correctly(self): flow = gf.Flow("test").add( - utils.SaveOrderTask('a', requires=['x']), + utils.ProgressingTask('a', requires=['x']), lf.Flow("test2", retry=retry.Times(2)).add( - utils.SaveOrderTask('b', provides='x'), + utils.ProgressingTask('b', provides='x'), utils.FailingTask('c'))) engine = self._make_engine(flow) engine.compile() @@ -669,26 +820,31 @@ class RetryTest(utils.EngineTestBase): engine.storage.save('test2_retry', 1) engine.storage.save('b', 11) engine.storage.save('a', 10) - self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) - self.assertItemsEqual(self.values[:3], [ - 'a reverted(10)', - 'c reverted(Failure: RuntimeError: Woot!)', - 'b reverted(11)', - ]) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) + expected = ['c.t RUNNING', + 'c.t FAILURE(Failure: RuntimeError: Woot!)', + 'a.t REVERTING', + 'c.t REVERTING', + 'a.t REVERTED', + 'c.t REVERTED', + 'b.t REVERTING', + 'b.t REVERTED'] + self.assertItemsEqual(capturer.values[:8], expected) # Task 'a' was or was not executed again, both cases are ok. - self.assertIsSuperAndSubsequence(self.values[3:], [ - 'b', - 'c reverted(Failure: RuntimeError: Woot!)', - 'b reverted(5)' + self.assertIsSuperAndSubsequence(capturer.values[8:], [ + 'b.t RUNNING', + 'c.t FAILURE(Failure: RuntimeError: Woot!)', + 'b.t REVERTED', ]) self.assertEqual(engine.storage.get_flow_state(), st.REVERTED) def test_nested_provides_graph_retried_correctly(self): flow = gf.Flow("test").add( - utils.SaveOrderTask('a', requires=['x']), + utils.ProgressingTask('a', requires=['x']), lf.Flow("test2", retry=retry.Times(2)).add( - utils.SaveOrderTask('b', provides='x'), - utils.SaveOrderTask('c'))) + utils.ProgressingTask('b', provides='x'), + utils.ProgressingTask('c'))) engine = self._make_engine(flow) engine.compile() engine.prepare() @@ -697,17 +853,31 @@ class RetryTest(utils.EngineTestBase): # pretend that 'c' failed fail = failure.Failure.from_exception(RuntimeError('Woot!')) engine.storage.save('c', fail, st.FAILURE) - - engine.run() - self.assertItemsEqual(self.values[:2], [ - 'c reverted(Failure: RuntimeError: Woot!)', - 'b reverted(11)', - ]) - self.assertItemsEqual(self.values[2:], ['b', 'c', 'a']) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + engine.run() + expected = ['c.t REVERTING', + 'c.t REVERTED', + 'b.t REVERTING', + 'b.t REVERTED'] + self.assertItemsEqual(capturer.values[:4], expected) + expected = ['test2_retry.r RETRYING', + 'b.t PENDING', + 'c.t PENDING', + 'test2_retry.r RUNNING', + 'test2_retry.r SUCCESS(2)', + 'b.t RUNNING', + 'b.t SUCCESS(5)', + 'a.t RUNNING', + 'c.t RUNNING', + 'a.t SUCCESS(5)', + 'c.t SUCCESS(5)'] + self.assertItemsEqual(expected, capturer.values[4:]) self.assertEqual(engine.storage.get_flow_state(), st.SUCCESS) class RetryParallelExecutionTest(utils.EngineTestBase): + # FIXME(harlowja): fix this class so that it doesn't use events or uses + # them in a way that works with more executors... def test_when_subflow_fails_revert_running_tasks(self): waiting_task = utils.WaitForOneFromTask('task1', 'task2', @@ -719,21 +889,35 @@ class RetryParallelExecutionTest(utils.EngineTestBase): engine = self._make_engine(flow) engine.task_notifier.register('*', waiting_task.callback) engine.storage.inject({'y': 2}) - engine.run() + with utils.CaptureListener(engine, capture_flow=False) as capturer: + engine.run() self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2}) - expected = ['task2', - 'task1', - u'task2 reverted(Failure: RuntimeError: Woot!)', - 'task1 reverted(5)', - 'task2', - 'task1'] - self.assertItemsEqual(self.values, expected) + expected = ['r.r RUNNING', + 'r.r SUCCESS(1)', + 'task1.t RUNNING', + 'task2.t RUNNING', + 'task2.t FAILURE(Failure: RuntimeError: Woot!)', + 'task2.t REVERTING', + 'task2.t REVERTED', + 'task1.t SUCCESS(5)', + 'task1.t REVERTING', + 'task1.t REVERTED', + 'r.r RETRYING', + 'task1.t PENDING', + 'task2.t PENDING', + 'r.r RUNNING', + 'r.r SUCCESS(2)', + 'task1.t RUNNING', + 'task2.t RUNNING', + 'task2.t SUCCESS(None)', + 'task1.t SUCCESS(5)'] + self.assertItemsEqual(capturer.values, expected) def test_when_subflow_fails_revert_success_tasks(self): waiting_task = utils.WaitForOneFromTask('task2', 'task1', [st.SUCCESS, st.FAILURE]) flow = uf.Flow('flow-1', retry.Times(3, 'r', provides='x')).add( - utils.SaveOrderTask('task1'), + utils.ProgressingTask('task1'), lf.Flow('flow-2').add( waiting_task, utils.ConditionalTask('task3')) @@ -741,22 +925,39 @@ class RetryParallelExecutionTest(utils.EngineTestBase): engine = self._make_engine(flow) engine.task_notifier.register('*', waiting_task.callback) engine.storage.inject({'y': 2}) - engine.run() + with utils.CaptureListener(engine, capture_flow=False) as capturer: + engine.run() self.assertEqual(engine.storage.fetch_all(), {'y': 2, 'x': 2}) - expected = ['task1', - 'task2', - 'task3', - u'task3 reverted(Failure: RuntimeError: Woot!)', - 'task1 reverted(5)', - 'task2 reverted(5)', - 'task1', - 'task2', - 'task3'] - self.assertItemsEqual(self.values, expected) + expected = ['r.r RUNNING', + 'r.r SUCCESS(1)', + 'task1.t RUNNING', + 'task2.t RUNNING', + 'task1.t SUCCESS(5)', + 'task2.t SUCCESS(5)', + 'task3.t RUNNING', + 'task3.t FAILURE(Failure: RuntimeError: Woot!)', + 'task3.t REVERTING', + 'task1.t REVERTING', + 'task3.t REVERTED', + 'task1.t REVERTED', + 'task2.t REVERTING', + 'task2.t REVERTED', + 'r.r RETRYING', + 'task1.t PENDING', + 'task2.t PENDING', + 'task3.t PENDING', + 'r.r RUNNING', + 'r.r SUCCESS(2)', + 'task1.t RUNNING', + 'task2.t RUNNING', + 'task1.t SUCCESS(5)', + 'task2.t SUCCESS(5)', + 'task3.t RUNNING', + 'task3.t SUCCESS(None)'] + self.assertItemsEqual(capturer.values, expected) -class SingleThreadedEngineTest(RetryTest, - test.TestCase): +class SerialEngineTest(RetryTest, test.TestCase): def _make_engine(self, flow, flow_detail=None): return taskflow.engines.load(flow, flow_detail=flow_detail, @@ -764,11 +965,41 @@ class SingleThreadedEngineTest(RetryTest, backend=self.backend) -class MultiThreadedEngineTest(RetryTest, - RetryParallelExecutionTest, - test.TestCase): +class ParallelEngineWithThreadsTest(RetryTest, + RetryParallelExecutionTest, + test.TestCase): + _EXECUTOR_WORKERS = 2 + def _make_engine(self, flow, flow_detail=None, executor=None): + if executor is None: + executor = 'threads' return taskflow.engines.load(flow, flow_detail=flow_detail, engine='parallel', backend=self.backend, + executor=executor, + max_workers=self._EXECUTOR_WORKERS) + + +@testtools.skipIf(not au.EVENTLET_AVAILABLE, 'eventlet is not available') +class ParallelEngineWithEventletTest(RetryTest, test.TestCase): + + def _make_engine(self, flow, flow_detail=None, executor=None): + if executor is None: + executor = futures.GreenThreadPoolExecutor() + self.addCleanup(executor.shutdown) + return taskflow.engines.load(flow, flow_detail=flow_detail, + backend=self.backend, engine='parallel', executor=executor) + + +class ParallelEngineWithProcessTest(RetryTest, test.TestCase): + _EXECUTOR_WORKERS = 2 + + def _make_engine(self, flow, flow_detail=None, executor=None): + if executor is None: + executor = 'processes' + return taskflow.engines.load(flow, flow_detail=flow_detail, + engine='parallel', + backend=self.backend, + executor=executor, + max_workers=self._EXECUTOR_WORKERS) diff --git a/taskflow/tests/unit/test_suspend.py b/taskflow/tests/unit/test_suspend.py new file mode 100644 index 00000000..08b2a83b --- /dev/null +++ b/taskflow/tests/unit/test_suspend.py @@ -0,0 +1,237 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import testtools + +import taskflow.engines +from taskflow import exceptions as exc +from taskflow.patterns import linear_flow as lf +from taskflow import states +from taskflow import test +from taskflow.tests import utils +from taskflow.types import futures +from taskflow.utils import async_utils as au + + +class SuspendingListener(utils.CaptureListener): + + def __init__(self, engine, + task_name, task_state, capture_flow=False): + super(SuspendingListener, self).__init__( + engine, + capture_flow=capture_flow) + self._revert_match = (task_name, task_state) + + def _task_receiver(self, state, details): + super(SuspendingListener, self)._task_receiver(state, details) + if (details['task_name'], state) == self._revert_match: + self._engine.suspend() + + +class SuspendTest(utils.EngineTestBase): + + def test_suspend_one_task(self): + flow = utils.ProgressingTask('a') + engine = self._make_engine(flow) + with SuspendingListener(engine, task_name='b', + task_state=states.SUCCESS) as capturer: + engine.run() + self.assertEqual(engine.storage.get_flow_state(), states.SUCCESS) + expected = ['a.t RUNNING', 'a.t SUCCESS(5)'] + self.assertEqual(expected, capturer.values) + with SuspendingListener(engine, task_name='b', + task_state=states.SUCCESS) as capturer: + engine.run() + self.assertEqual(engine.storage.get_flow_state(), states.SUCCESS) + expected = [] + self.assertEqual(expected, capturer.values) + + def test_suspend_linear_flow(self): + flow = lf.Flow('linear').add( + utils.ProgressingTask('a'), + utils.ProgressingTask('b'), + utils.ProgressingTask('c') + ) + engine = self._make_engine(flow) + with SuspendingListener(engine, task_name='b', + task_state=states.SUCCESS) as capturer: + engine.run() + self.assertEqual(engine.storage.get_flow_state(), states.SUSPENDED) + expected = ['a.t RUNNING', 'a.t SUCCESS(5)', + 'b.t RUNNING', 'b.t SUCCESS(5)'] + self.assertEqual(expected, capturer.values) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + engine.run() + self.assertEqual(engine.storage.get_flow_state(), states.SUCCESS) + expected = ['c.t RUNNING', 'c.t SUCCESS(5)'] + self.assertEqual(expected, capturer.values) + + def test_suspend_linear_flow_on_revert(self): + flow = lf.Flow('linear').add( + utils.ProgressingTask('a'), + utils.ProgressingTask('b'), + utils.FailingTask('c') + ) + engine = self._make_engine(flow) + with SuspendingListener(engine, task_name='b', + task_state=states.REVERTED) as capturer: + engine.run() + self.assertEqual(engine.storage.get_flow_state(), states.SUSPENDED) + expected = ['a.t RUNNING', + 'a.t SUCCESS(5)', + 'b.t RUNNING', + 'b.t SUCCESS(5)', + 'c.t RUNNING', + 'c.t FAILURE(Failure: RuntimeError: Woot!)', + 'c.t REVERTING', + 'c.t REVERTED', + 'b.t REVERTING', + 'b.t REVERTED'] + self.assertEqual(expected, capturer.values) + with utils.CaptureListener(engine, capture_flow=False) as capturer: + self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) + self.assertEqual(engine.storage.get_flow_state(), states.REVERTED) + expected = ['a.t REVERTING', 'a.t REVERTED'] + self.assertEqual(expected, capturer.values) + + def test_suspend_and_resume_linear_flow_on_revert(self): + flow = lf.Flow('linear').add( + utils.ProgressingTask('a'), + utils.ProgressingTask('b'), + utils.FailingTask('c') + ) + engine = self._make_engine(flow) + with SuspendingListener(engine, task_name='b', + task_state=states.REVERTED) as capturer: + engine.run() + expected = ['a.t RUNNING', + 'a.t SUCCESS(5)', + 'b.t RUNNING', + 'b.t SUCCESS(5)', + 'c.t RUNNING', + 'c.t FAILURE(Failure: RuntimeError: Woot!)', + 'c.t REVERTING', + 'c.t REVERTED', + 'b.t REVERTING', + 'b.t REVERTED'] + self.assertEqual(expected, capturer.values) + + # pretend we are resuming + engine2 = self._make_engine(flow, engine.storage._flowdetail) + with utils.CaptureListener(engine2, capture_flow=False) as capturer2: + self.assertRaisesRegexp(RuntimeError, '^Woot', engine2.run) + self.assertEqual(engine2.storage.get_flow_state(), states.REVERTED) + expected = ['a.t REVERTING', + 'a.t REVERTED'] + self.assertEqual(expected, capturer2.values) + + def test_suspend_and_revert_even_if_task_is_gone(self): + flow = lf.Flow('linear').add( + utils.ProgressingTask('a'), + utils.ProgressingTask('b'), + utils.FailingTask('c') + ) + engine = self._make_engine(flow) + + with SuspendingListener(engine, task_name='b', + task_state=states.REVERTED) as capturer: + engine.run() + + expected = ['a.t RUNNING', + 'a.t SUCCESS(5)', + 'b.t RUNNING', + 'b.t SUCCESS(5)', + 'c.t RUNNING', + 'c.t FAILURE(Failure: RuntimeError: Woot!)', + 'c.t REVERTING', + 'c.t REVERTED', + 'b.t REVERTING', + 'b.t REVERTED'] + self.assertEqual(expected, capturer.values) + + # pretend we are resuming, but task 'c' gone when flow got updated + flow2 = lf.Flow('linear').add( + utils.ProgressingTask('a'), + utils.ProgressingTask('b'), + ) + engine2 = self._make_engine(flow2, engine.storage._flowdetail) + with utils.CaptureListener(engine2, capture_flow=False) as capturer2: + self.assertRaisesRegexp(RuntimeError, '^Woot', engine2.run) + self.assertEqual(engine2.storage.get_flow_state(), states.REVERTED) + expected = ['a.t REVERTING', 'a.t REVERTED'] + self.assertEqual(capturer2.values, expected) + + def test_storage_is_rechecked(self): + flow = lf.Flow('linear').add( + utils.ProgressingTask('b', requires=['foo']), + utils.ProgressingTask('c') + ) + engine = self._make_engine(flow) + engine.storage.inject({'foo': 'bar'}) + with SuspendingListener(engine, task_name='b', + task_state=states.SUCCESS): + engine.run() + self.assertEqual(engine.storage.get_flow_state(), states.SUSPENDED) + # uninject everything: + engine.storage.save(engine.storage.injector_name, + {}, states.SUCCESS) + self.assertRaises(exc.MissingDependencies, engine.run) + + +class SerialEngineTest(SuspendTest, test.TestCase): + def _make_engine(self, flow, flow_detail=None): + return taskflow.engines.load(flow, + flow_detail=flow_detail, + engine='serial', + backend=self.backend) + + +class ParallelEngineWithThreadsTest(SuspendTest, test.TestCase): + _EXECUTOR_WORKERS = 2 + + def _make_engine(self, flow, flow_detail=None, executor=None): + if executor is None: + executor = 'threads' + return taskflow.engines.load(flow, flow_detail=flow_detail, + engine='parallel', + backend=self.backend, + executor=executor, + max_workers=self._EXECUTOR_WORKERS) + + +@testtools.skipIf(not au.EVENTLET_AVAILABLE, 'eventlet is not available') +class ParallelEngineWithEventletTest(SuspendTest, test.TestCase): + + def _make_engine(self, flow, flow_detail=None, executor=None): + if executor is None: + executor = futures.GreenThreadPoolExecutor() + self.addCleanup(executor.shutdown) + return taskflow.engines.load(flow, flow_detail=flow_detail, + backend=self.backend, engine='parallel', + executor=executor) + + +class ParallelEngineWithProcessTest(SuspendTest, test.TestCase): + _EXECUTOR_WORKERS = 2 + + def _make_engine(self, flow, flow_detail=None, executor=None): + if executor is None: + executor = 'processes' + return taskflow.engines.load(flow, flow_detail=flow_detail, + engine='parallel', + backend=self.backend, + executor=executor, + max_workers=self._EXECUTOR_WORKERS) diff --git a/taskflow/tests/unit/test_suspend_flow.py b/taskflow/tests/unit/test_suspend_flow.py deleted file mode 100644 index 928f2bec..00000000 --- a/taskflow/tests/unit/test_suspend_flow.py +++ /dev/null @@ -1,195 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import testtools - -import taskflow.engines -from taskflow import exceptions as exc -from taskflow.listeners import base as lbase -from taskflow.patterns import linear_flow as lf -from taskflow import states -from taskflow import test -from taskflow.tests import utils -from taskflow.types import futures -from taskflow.utils import async_utils as au - - -class SuspendingListener(lbase.ListenerBase): - - def __init__(self, engine, task_name, task_state): - super(SuspendingListener, self).__init__( - engine, task_listen_for=(task_state,)) - self._task_name = task_name - - def _task_receiver(self, state, details): - if details['task_name'] == self._task_name: - self._engine.suspend() - - -class SuspendFlowTest(utils.EngineTestBase): - - def test_suspend_one_task(self): - flow = utils.SaveOrderTask('a') - engine = self._make_engine(flow) - with SuspendingListener(engine, task_name='b', - task_state=states.SUCCESS): - engine.run() - self.assertEqual(engine.storage.get_flow_state(), states.SUCCESS) - self.assertEqual(self.values, ['a']) - engine.run() - self.assertEqual(engine.storage.get_flow_state(), states.SUCCESS) - self.assertEqual(self.values, ['a']) - - def test_suspend_linear_flow(self): - flow = lf.Flow('linear').add( - utils.SaveOrderTask('a'), - utils.SaveOrderTask('b'), - utils.SaveOrderTask('c') - ) - engine = self._make_engine(flow) - with SuspendingListener(engine, task_name='b', - task_state=states.SUCCESS): - engine.run() - self.assertEqual(engine.storage.get_flow_state(), states.SUSPENDED) - self.assertEqual(self.values, ['a', 'b']) - engine.run() - self.assertEqual(engine.storage.get_flow_state(), states.SUCCESS) - self.assertEqual(self.values, ['a', 'b', 'c']) - - def test_suspend_linear_flow_on_revert(self): - flow = lf.Flow('linear').add( - utils.SaveOrderTask('a'), - utils.SaveOrderTask('b'), - utils.FailingTask('c') - ) - engine = self._make_engine(flow) - with SuspendingListener(engine, task_name='b', - task_state=states.REVERTED): - engine.run() - self.assertEqual(engine.storage.get_flow_state(), states.SUSPENDED) - self.assertEqual( - self.values, - ['a', 'b', - 'c reverted(Failure: RuntimeError: Woot!)', - 'b reverted(5)']) - self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run) - self.assertEqual(engine.storage.get_flow_state(), states.REVERTED) - self.assertEqual( - self.values, - ['a', - 'b', - 'c reverted(Failure: RuntimeError: Woot!)', - 'b reverted(5)', - 'a reverted(5)']) - - def test_suspend_and_resume_linear_flow_on_revert(self): - flow = lf.Flow('linear').add( - utils.SaveOrderTask('a'), - utils.SaveOrderTask('b'), - utils.FailingTask('c') - ) - engine = self._make_engine(flow) - - with SuspendingListener(engine, task_name='b', - task_state=states.REVERTED): - engine.run() - - # pretend we are resuming - engine2 = self._make_engine(flow, engine.storage._flowdetail) - self.assertRaisesRegexp(RuntimeError, '^Woot', engine2.run) - self.assertEqual(engine2.storage.get_flow_state(), states.REVERTED) - self.assertEqual( - self.values, - ['a', - 'b', - 'c reverted(Failure: RuntimeError: Woot!)', - 'b reverted(5)', - 'a reverted(5)']) - - def test_suspend_and_revert_even_if_task_is_gone(self): - flow = lf.Flow('linear').add( - utils.SaveOrderTask('a'), - utils.SaveOrderTask('b'), - utils.FailingTask('c') - ) - engine = self._make_engine(flow) - - with SuspendingListener(engine, task_name='b', - task_state=states.REVERTED): - engine.run() - - expected_values = ['a', 'b', - 'c reverted(Failure: RuntimeError: Woot!)', - 'b reverted(5)'] - self.assertEqual(self.values, expected_values) - - # pretend we are resuming, but task 'c' gone when flow got updated - flow2 = lf.Flow('linear').add( - utils.SaveOrderTask('a'), - utils.SaveOrderTask('b') - ) - engine2 = self._make_engine(flow2, engine.storage._flowdetail) - self.assertRaisesRegexp(RuntimeError, '^Woot', engine2.run) - self.assertEqual(engine2.storage.get_flow_state(), states.REVERTED) - expected_values.append('a reverted(5)') - self.assertEqual(self.values, expected_values) - - def test_storage_is_rechecked(self): - flow = lf.Flow('linear').add( - utils.SaveOrderTask('b', requires=['foo']), - utils.SaveOrderTask('c') - ) - engine = self._make_engine(flow) - engine.storage.inject({'foo': 'bar'}) - with SuspendingListener(engine, task_name='b', - task_state=states.SUCCESS): - engine.run() - self.assertEqual(engine.storage.get_flow_state(), states.SUSPENDED) - # uninject everything: - engine.storage.save(engine.storage.injector_name, - {}, states.SUCCESS) - self.assertRaises(exc.MissingDependencies, engine.run) - - -class SingleThreadedEngineTest(SuspendFlowTest, - test.TestCase): - def _make_engine(self, flow, flow_detail=None): - return taskflow.engines.load(flow, - flow_detail=flow_detail, - engine='serial', - backend=self.backend) - - -class MultiThreadedEngineTest(SuspendFlowTest, - test.TestCase): - def _make_engine(self, flow, flow_detail=None, executor=None): - return taskflow.engines.load(flow, flow_detail=flow_detail, - engine='parallel', - backend=self.backend, - executor=executor) - - -@testtools.skipIf(not au.EVENTLET_AVAILABLE, 'eventlet is not available') -class ParallelEngineWithEventletTest(SuspendFlowTest, - test.TestCase): - - def _make_engine(self, flow, flow_detail=None, executor=None): - if executor is None: - executor = futures.GreenThreadPoolExecutor() - return taskflow.engines.load(flow, flow_detail=flow_detail, - engine='parallel', - backend=self.backend, - executor=executor) diff --git a/taskflow/tests/unit/worker_based/test_worker.py b/taskflow/tests/unit/worker_based/test_worker.py index 8fc76eb4..e1661353 100644 --- a/taskflow/tests/unit/worker_based/test_worker.py +++ b/taskflow/tests/unit/worker_based/test_worker.py @@ -34,7 +34,7 @@ class TestWorker(test.MockTestCase): self.exchange = 'test-exchange' self.topic = 'test-topic' self.threads_count = 5 - self.endpoint_count = 22 + self.endpoint_count = 21 # patch classes self.executor_mock, self.executor_inst_mock = self.patchClass( diff --git a/taskflow/tests/utils.py b/taskflow/tests/utils.py index a0b2ff0d..5abdd100 100644 --- a/taskflow/tests/utils.py +++ b/taskflow/tests/utils.py @@ -20,6 +20,7 @@ import string import six from taskflow import exceptions +from taskflow.listeners import base as listener_base from taskflow.persistence.backends import impl_memory from taskflow import retry from taskflow import task @@ -116,43 +117,71 @@ class ProvidesRequiresTask(task.Task): return dict((k, k) for k in self.provides) -def task_callback(state, values, details): - name = details.get('task_name', None) - if not name: - name = details.get('retry_name', '') - values.append('%s %s' % (name, state)) +class CaptureListener(listener_base.Listener): + _LOOKUP_NAME_POSTFIX = { + 'task_name': '.t', + 'retry_name': '.r', + 'flow_name': '.f', + } + + def __init__(self, engine, + task_listen_for=listener_base.DEFAULT_LISTEN_FOR, + values=None, + capture_flow=True, capture_task=True, capture_retry=True, + skip_tasks=None, skip_retries=None, skip_flows=None): + super(CaptureListener, self).__init__(engine, + task_listen_for=task_listen_for) + self._capture_flow = capture_flow + self._capture_task = capture_task + self._capture_retry = capture_retry + self._skip_tasks = skip_tasks or [] + self._skip_flows = skip_flows or [] + self._skip_retries = skip_retries or [] + if values is None: + self.values = [] + else: + self.values = values + + def _capture(self, state, details, name_key): + name = details[name_key] + try: + name += self._LOOKUP_NAME_POSTFIX[name_key] + except KeyError: + pass + if 'result' in details: + name += ' %s(%s)' % (state, details['result']) + else: + name += " %s" % state + return name + + def _task_receiver(self, state, details): + if self._capture_task: + if details['task_name'] not in self._skip_tasks: + self.values.append(self._capture(state, details, 'task_name')) + + def _retry_receiver(self, state, details): + if self._capture_retry: + if details['retry_name'] not in self._skip_retries: + self.values.append(self._capture(state, details, 'retry_name')) + + def _flow_receiver(self, state, details): + if self._capture_flow: + if details['flow_name'] not in self._skip_flows: + self.values.append(self._capture(state, details, 'flow_name')) -def flow_callback(state, values, details): - values.append('flow %s' % state) - - -def register_notifiers(engine, values): - engine.notifier.register('*', flow_callback, kwargs={'values': values}) - engine.task_notifier.register('*', task_callback, - kwargs={'values': values}) - - -class SaveOrderTask(task.Task): - - def __init__(self, name=None, *args, **kwargs): - super(SaveOrderTask, self).__init__(name=name, *args, **kwargs) - self.values = EngineTestBase.values - +class ProgressingTask(task.Task): def execute(self, **kwargs): self.update_progress(0.0) - self.values.append(self.name) self.update_progress(1.0) return 5 def revert(self, **kwargs): self.update_progress(0) - self.values.append(self.name + ' reverted(%s)' - % kwargs.get('result')) self.update_progress(1.0) -class FailingTask(SaveOrderTask): +class FailingTask(ProgressingTask): def execute(self, **kwargs): self.update_progress(0) self.update_progress(0.99) @@ -173,7 +202,7 @@ class ProgressingTask(task.Task): return 5 -class FailingTaskWithOneArg(SaveOrderTask): +class FailingTaskWithOneArg(ProgressingTask): def execute(self, x, **kwargs): raise RuntimeError('Woot with %s' % x) @@ -282,11 +311,8 @@ class NeverRunningTask(task.Task): class EngineTestBase(object): - values = None - def setUp(self): super(EngineTestBase, self).setUp() - EngineTestBase.values = [] self.backend = impl_memory.MemoryBackend(conf={}) def tearDown(self): @@ -324,7 +350,7 @@ class OneReturnRetry(retry.AlwaysRevert): pass -class ConditionalTask(SaveOrderTask): +class ConditionalTask(ProgressingTask): def execute(self, x, y): super(ConditionalTask, self).execute() @@ -332,7 +358,7 @@ class ConditionalTask(SaveOrderTask): raise RuntimeError('Woot!') -class WaitForOneFromTask(SaveOrderTask): +class WaitForOneFromTask(ProgressingTask): def __init__(self, name, wait_for, wait_states, **kwargs): super(WaitForOneFromTask, self).__init__(name, **kwargs) diff --git a/taskflow/types/futures.py b/taskflow/types/futures.py index 8e8e67af..8c3d5504 100644 --- a/taskflow/types/futures.py +++ b/taskflow/types/futures.py @@ -113,6 +113,7 @@ class ThreadPoolExecutor(_thread.ThreadPoolExecutor): @property def alive(self): + """Accessor to determine if the executor is alive/active.""" return not self._shutdown def submit(self, fn, *args, **kwargs): @@ -141,6 +142,7 @@ class ProcessPoolExecutor(_process.ProcessPoolExecutor): @property def alive(self): + """Accessor to determine if the executor is alive/active.""" return not self._shutdown_thread @property @@ -189,6 +191,7 @@ class SynchronousExecutor(_futures.Executor): @property def alive(self): + """Accessor to determine if the executor is alive/active.""" return not self._shutoff def shutdown(self, wait=True): @@ -276,6 +279,7 @@ class GreenThreadPoolExecutor(_futures.Executor): @property def alive(self): + """Accessor to determine if the executor is alive/active.""" return not self._shutdown @property From 1ed0f22fd35c4e08667f21bf549b3fe84a230d32 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 15 Jan 2015 14:52:20 -0800 Subject: [PATCH 202/240] Use constants for runner state machine event names Instead of using strings it is better if we can use constants (that may be the same/adjusted strings) and use those instead in the state machine used in the runner. The names are adjusted (and the state graph diagram and docstring) to reflect names that fit better with there intended meaning and usage. Change-Id: Iaf229d6e37730545ba9f2708d118697cb7145992 --- doc/source/img/engine_states.svg | 4 +- taskflow/engines/action_engine/runner.py | 103 +++++++++++++---------- tools/state_graph.py | 9 +- 3 files changed, 67 insertions(+), 49 deletions(-) diff --git a/doc/source/img/engine_states.svg b/doc/source/img/engine_states.svg index 807b8ea5..08a419d5 100644 --- a/doc/source/img/engine_states.svg +++ b/doc/source/img/engine_states.svg @@ -3,6 +3,6 @@ - -Engines statesGAME_OVERREVERTEDon revertedSUCCESSon successSUSPENDEDon suspendedFAILUREon failedUNDEFINEDRESUMINGon startSCHEDULINGon scheduleANALYZINGon finishedon scheduleWAITINGon waiton waiton analyzestart + +Engines statesGAME_OVERREVERTEDrevertedSUCCESSsuccessSUSPENDEDsuspendedFAILUREfailedUNDEFINEDRESUMINGstartSCHEDULINGschedule nextANALYZINGcompletedschedule nextWAITINGwait finishedwait finishedexamine finishedstart diff --git a/taskflow/engines/action_engine/runner.py b/taskflow/engines/action_engine/runner.py index 6d610569..45ee0ba6 100644 --- a/taskflow/engines/action_engine/runner.py +++ b/taskflow/engines/action_engine/runner.py @@ -27,6 +27,17 @@ _UNDEFINED = 'UNDEFINED' _GAME_OVER = 'GAME_OVER' _META_STATES = (_GAME_OVER, _UNDEFINED) +# Event name constants the state machine uses. +_SCHEDULE = 'schedule_next' +_WAIT = 'wait_finished' +_ANALYZE = 'examine_finished' +_FINISH = 'completed' +_FAILED = 'failed' +_SUSPENDED = 'suspended' +_SUCCESS = 'success' +_REVERTED = 'reverted' +_START = 'start' + LOG = logging.getLogger(__name__) @@ -45,25 +56,25 @@ class _MachineBuilder(object): NOTE(harlowja): the machine states that this build will for are:: - +--------------+-----------+------------+----------+---------+ - Start | Event | End | On Enter | On Exit - +--------------+-----------+------------+----------+---------+ - ANALYZING | finished | GAME_OVER | | - ANALYZING | schedule | SCHEDULING | | - ANALYZING | wait | WAITING | | - FAILURE[$] | | | | - GAME_OVER | failed | FAILURE | | - GAME_OVER | reverted | REVERTED | | - GAME_OVER | success | SUCCESS | | - GAME_OVER | suspended | SUSPENDED | | - RESUMING | schedule | SCHEDULING | | - REVERTED[$] | | | | - SCHEDULING | wait | WAITING | | - SUCCESS[$] | | | | - SUSPENDED[$] | | | | - UNDEFINED[^] | start | RESUMING | | - WAITING | analyze | ANALYZING | | - +--------------+-----------+------------+----------+---------+ + +--------------+------------------+------------+----------+---------+ + Start | Event | End | On Enter | On Exit + +--------------+------------------+------------+----------+---------+ + ANALYZING | completed | GAME_OVER | | + ANALYZING | schedule_next | SCHEDULING | | + ANALYZING | wait_finished | WAITING | | + FAILURE[$] | | | | + GAME_OVER | failed | FAILURE | | + GAME_OVER | reverted | REVERTED | | + GAME_OVER | success | SUCCESS | | + GAME_OVER | suspended | SUSPENDED | | + RESUMING | schedule_next | SCHEDULING | | + REVERTED[$] | | | | + SCHEDULING | wait_finished | WAITING | | + SUCCESS[$] | | | | + SUSPENDED[$] | | | | + UNDEFINED[^] | start | RESUMING | | + WAITING | examine_finished | ANALYZING | | + +--------------+------------------+------------+----------+---------+ Between any of these yielded states (minus ``GAME_OVER`` and ``UNDEFINED``) if the engine has been suspended or the engine has failed (due to a @@ -90,17 +101,17 @@ class _MachineBuilder(object): def resume(old_state, new_state, event): memory.next_nodes.update(self._completer.resume()) memory.next_nodes.update(self._analyzer.get_next_nodes()) - return 'schedule' + return _SCHEDULE def game_over(old_state, new_state, event): if memory.failures: - return 'failed' + return _FAILED if self._analyzer.get_next_nodes(): - return 'suspended' + return _SUSPENDED elif self._analyzer.is_success(): - return 'success' + return _SUCCESS else: - return 'reverted' + return _REVERTED def schedule(old_state, new_state, event): if self.runnable() and memory.next_nodes: @@ -111,7 +122,7 @@ class _MachineBuilder(object): if failures: memory.failures.extend(failures) memory.next_nodes.clear() - return 'wait' + return _WAIT def wait(old_state, new_state, event): # TODO(harlowja): maybe we should start doing 'yield from' this @@ -122,7 +133,7 @@ class _MachineBuilder(object): timeout) memory.done.update(done) memory.not_done = not_done - return 'analyze' + return _ANALYZE def analyze(old_state, new_state, event): next_nodes = set() @@ -145,11 +156,11 @@ class _MachineBuilder(object): next_nodes.update(more_nodes) if self.runnable() and next_nodes and not memory.failures: memory.next_nodes.update(next_nodes) - return 'schedule' + return _SCHEDULE elif memory.not_done: - return 'wait' + return _WAIT else: - return 'finished' + return _FINISH def on_exit(old_state, event): LOG.debug("Exiting old state '%s' in response to event '%s'", @@ -178,23 +189,23 @@ class _MachineBuilder(object): m.add_state(st.WAITING, **watchers) m.add_state(st.FAILURE, terminal=True, **watchers) - m.add_transition(_GAME_OVER, st.REVERTED, 'reverted') - m.add_transition(_GAME_OVER, st.SUCCESS, 'success') - m.add_transition(_GAME_OVER, st.SUSPENDED, 'suspended') - m.add_transition(_GAME_OVER, st.FAILURE, 'failed') - m.add_transition(_UNDEFINED, st.RESUMING, 'start') - m.add_transition(st.ANALYZING, _GAME_OVER, 'finished') - m.add_transition(st.ANALYZING, st.SCHEDULING, 'schedule') - m.add_transition(st.ANALYZING, st.WAITING, 'wait') - m.add_transition(st.RESUMING, st.SCHEDULING, 'schedule') - m.add_transition(st.SCHEDULING, st.WAITING, 'wait') - m.add_transition(st.WAITING, st.ANALYZING, 'analyze') + m.add_transition(_GAME_OVER, st.REVERTED, _REVERTED) + m.add_transition(_GAME_OVER, st.SUCCESS, _SUCCESS) + m.add_transition(_GAME_OVER, st.SUSPENDED, _SUSPENDED) + m.add_transition(_GAME_OVER, st.FAILURE, _FAILED) + m.add_transition(_UNDEFINED, st.RESUMING, _START) + m.add_transition(st.ANALYZING, _GAME_OVER, _FINISH) + m.add_transition(st.ANALYZING, st.SCHEDULING, _SCHEDULE) + m.add_transition(st.ANALYZING, st.WAITING, _WAIT) + m.add_transition(st.RESUMING, st.SCHEDULING, _SCHEDULE) + m.add_transition(st.SCHEDULING, st.WAITING, _WAIT) + m.add_transition(st.WAITING, st.ANALYZING, _ANALYZE) - m.add_reaction(_GAME_OVER, 'finished', game_over) - m.add_reaction(st.ANALYZING, 'analyze', analyze) - m.add_reaction(st.RESUMING, 'start', resume) - m.add_reaction(st.SCHEDULING, 'schedule', schedule) - m.add_reaction(st.WAITING, 'wait', wait) + m.add_reaction(_GAME_OVER, _FINISH, game_over) + m.add_reaction(st.ANALYZING, _ANALYZE, analyze) + m.add_reaction(st.RESUMING, _START, resume) + m.add_reaction(st.SCHEDULING, _SCHEDULE, schedule) + m.add_reaction(st.WAITING, _WAIT, wait) m.freeze() return (m, memory) @@ -231,7 +242,7 @@ class Runner(object): def run_iter(self, timeout=None): """Runs the nodes using a built state machine.""" machine, memory = self.builder.build(timeout=timeout) - for (_prior_state, new_state) in machine.run_iter('start'): + for (_prior_state, new_state) in machine.run_iter(_START): # NOTE(harlowja): skip over meta-states. if new_state not in _META_STATES: if new_state == st.FAILURE: diff --git a/tools/state_graph.py b/tools/state_graph.py index b4f9d53b..ce5a1d79 100755 --- a/tools/state_graph.py +++ b/tools/state_graph.py @@ -41,6 +41,12 @@ class DummyRuntime(object): self.storage = None +def clean_event(name): + name = name.replace("_", " ") + name = name.strip() + return name + + def make_machine(start_state, transitions, disallowed): machine = fsm.FSM(start_state) machine.add_state(start_state) @@ -129,6 +135,7 @@ def main(): } nodes = {} for (start_state, on_event, end_state) in source: + on_event = clean_event(on_event) if start_state not in nodes: start_node_attrs = node_attrs.copy() text_color = map_color(internal_states, start_state) @@ -145,7 +152,7 @@ def main(): g.add_node(nodes[end_state]) if options.engines: edge_attrs = { - 'label': "on %s" % on_event + 'label': on_event, } if 'reverted' in on_event: edge_attrs['fontcolor'] = 'darkorange' From d92c226fe2d4a4fc402b96354da29dcc22a25574 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 11 Jan 2015 00:15:09 -0800 Subject: [PATCH 203/240] Add back a 'eventlet_utils' helper utility module Recreate a very simple eventlet utility module that has only a few features; one function checks if eventlet is available and if not raise an exception; and a constant that can be used by calling code (such as tests or other optional functionality) to check if eventlet is useable before proceeding. Change-Id: I32df0702eeae7c7c78972c9796156dd824b2f123 --- doc/source/utils.rst | 5 +++ taskflow/examples/hello_world.py | 9 ++--- taskflow/examples/parallel_table_multiply.py | 4 +-- taskflow/examples/resume_vm_boot.py | 4 +-- .../persistence/backends/impl_sqlalchemy.py | 4 +-- .../tests/unit/action_engine/test_creation.py | 4 +-- taskflow/tests/unit/test_engines.py | 4 +-- taskflow/tests/unit/test_futures.py | 6 ++-- taskflow/tests/unit/test_retries.py | 4 +-- taskflow/tests/unit/test_suspend.py | 4 +-- taskflow/tests/unit/test_utils_async_utils.py | 3 +- taskflow/types/futures.py | 10 +++--- taskflow/utils/async_utils.py | 7 ++-- taskflow/utils/eventlet_utils.py | 34 +++++++++++++++++++ 14 files changed, 70 insertions(+), 32 deletions(-) create mode 100644 taskflow/utils/eventlet_utils.py diff --git a/doc/source/utils.rst b/doc/source/utils.rst index 8125aac6..d3c726f0 100644 --- a/doc/source/utils.rst +++ b/doc/source/utils.rst @@ -18,6 +18,11 @@ Deprecation .. automodule:: taskflow.utils.deprecation +Eventlet +~~~~~~~~ + +.. automodule:: taskflow.utils.eventlet_utils + Kazoo ~~~~~ diff --git a/taskflow/examples/hello_world.py b/taskflow/examples/hello_world.py index 22f6a3b0..f8e0bb23 100644 --- a/taskflow/examples/hello_world.py +++ b/taskflow/examples/hello_world.py @@ -25,17 +25,12 @@ top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) sys.path.insert(0, top_dir) -try: - import eventlet # noqa - EVENTLET_AVAILABLE = True -except ImportError: - EVENTLET_AVAILABLE = False - from taskflow import engines from taskflow.patterns import linear_flow as lf from taskflow.patterns import unordered_flow as uf from taskflow import task from taskflow.types import futures +from taskflow.utils import eventlet_utils # INTRO: This is the defacto hello world equivalent for taskflow; it shows how @@ -86,7 +81,7 @@ song.add(PrinterTask("conductor@begin", show_name=False, inject={'output': "*dong*"})) # Run in parallel using eventlet green threads... -if EVENTLET_AVAILABLE: +if eventlet_utils.EVENTLET_AVAILABLE: with futures.GreenThreadPoolExecutor() as executor: e = engines.load(song, executor=executor, engine='parallel') e.run() diff --git a/taskflow/examples/parallel_table_multiply.py b/taskflow/examples/parallel_table_multiply.py index 88562a2e..f4550c20 100644 --- a/taskflow/examples/parallel_table_multiply.py +++ b/taskflow/examples/parallel_table_multiply.py @@ -33,7 +33,7 @@ from taskflow import engines from taskflow.patterns import unordered_flow as uf from taskflow import task from taskflow.types import futures -from taskflow.utils import async_utils +from taskflow.utils import eventlet_utils # INTRO: This example walks through a miniature workflow which does a parallel # table modification where each row in the table gets adjusted by a thread, or @@ -97,7 +97,7 @@ def main(): f = make_flow(tbl) # Now run it (using the specified executor)... - if async_utils.EVENTLET_AVAILABLE: + if eventlet_utils.EVENTLET_AVAILABLE: executor = futures.GreenThreadPoolExecutor(max_workers=5) else: executor = futures.ThreadPoolExecutor(max_workers=5) diff --git a/taskflow/examples/resume_vm_boot.py b/taskflow/examples/resume_vm_boot.py index f400d0d1..7d34a95a 100644 --- a/taskflow/examples/resume_vm_boot.py +++ b/taskflow/examples/resume_vm_boot.py @@ -39,7 +39,7 @@ from taskflow.patterns import graph_flow as gf from taskflow.patterns import linear_flow as lf from taskflow import task from taskflow.types import futures -from taskflow.utils import async_utils +from taskflow.utils import eventlet_utils from taskflow.utils import persistence_utils as p_utils import example_utils as eu # noqa @@ -238,7 +238,7 @@ with eu.get_backend() as backend: # Set up how we want our engine to run, serial, parallel... executor = None - if async_utils.EVENTLET_AVAILABLE: + if eventlet_utils.EVENTLET_AVAILABLE: executor = futures.GreenThreadPoolExecutor(5) # Create/fetch a logbook that will track the workflows work. diff --git a/taskflow/persistence/backends/impl_sqlalchemy.py b/taskflow/persistence/backends/impl_sqlalchemy.py index c65c9134..1ae09f61 100644 --- a/taskflow/persistence/backends/impl_sqlalchemy.py +++ b/taskflow/persistence/backends/impl_sqlalchemy.py @@ -38,7 +38,7 @@ from taskflow.persistence.backends.sqlalchemy import migration from taskflow.persistence.backends.sqlalchemy import models from taskflow.persistence import logbook from taskflow.types import failure -from taskflow.utils import async_utils +from taskflow.utils import eventlet_utils from taskflow.utils import misc @@ -250,7 +250,7 @@ class SQLAlchemyBackend(base.Backend): engine_args.update(conf.pop('engine_args', {})) engine = sa.create_engine(sql_connection, **engine_args) checkin_yield = conf.pop('checkin_yield', - async_utils.EVENTLET_AVAILABLE) + eventlet_utils.EVENTLET_AVAILABLE) if _as_bool(checkin_yield): sa.event.listen(engine, 'checkin', _thread_yield) if 'mysql' in e_url.drivername: diff --git a/taskflow/tests/unit/action_engine/test_creation.py b/taskflow/tests/unit/action_engine/test_creation.py index c8a0b436..2c6ed585 100644 --- a/taskflow/tests/unit/action_engine/test_creation.py +++ b/taskflow/tests/unit/action_engine/test_creation.py @@ -23,7 +23,7 @@ from taskflow.persistence import backends from taskflow import test from taskflow.tests import utils from taskflow.types import futures as futures -from taskflow.utils import async_utils as au +from taskflow.utils import eventlet_utils as eu from taskflow.utils import persistence_utils as pu @@ -61,7 +61,7 @@ class ParallelCreationTest(test.TestCase): self.assertIsInstance(eng._task_executor, executor.ParallelProcessTaskExecutor) - @testtools.skipIf(not au.EVENTLET_AVAILABLE, 'eventlet is not available') + @testtools.skipIf(not eu.EVENTLET_AVAILABLE, 'eventlet is not available') def test_green_executor_creation(self): with futures.GreenThreadPoolExecutor(1) as e: eng = self._create_engine(executor=e) diff --git a/taskflow/tests/unit/test_engines.py b/taskflow/tests/unit/test_engines.py index 6865bb82..8762d386 100644 --- a/taskflow/tests/unit/test_engines.py +++ b/taskflow/tests/unit/test_engines.py @@ -34,7 +34,7 @@ from taskflow.tests import utils from taskflow.types import failure from taskflow.types import futures from taskflow.types import graph as gr -from taskflow.utils import async_utils as au +from taskflow.utils import eventlet_utils as eu from taskflow.utils import persistence_utils as p_utils from taskflow.utils import threading_utils as tu @@ -652,7 +652,7 @@ class ParallelEngineWithThreadsTest(EngineTaskTest, executor.shutdown(wait=True) -@testtools.skipIf(not au.EVENTLET_AVAILABLE, 'eventlet is not available') +@testtools.skipIf(not eu.EVENTLET_AVAILABLE, 'eventlet is not available') class ParallelEngineWithEventletTest(EngineTaskTest, EngineLinearFlowTest, EngineParallelFlowTest, diff --git a/taskflow/tests/unit/test_futures.py b/taskflow/tests/unit/test_futures.py index 437bf28a..ce2c69c1 100644 --- a/taskflow/tests/unit/test_futures.py +++ b/taskflow/tests/unit/test_futures.py @@ -23,13 +23,13 @@ import testtools from taskflow import test from taskflow.types import futures +from taskflow.utils import eventlet_utils as eu try: from eventlet.green import threading as greenthreading from eventlet.green import time as greentime - EVENTLET_AVAILABLE = True except ImportError: - EVENTLET_AVAILABLE = False + pass def _noop(): @@ -194,7 +194,7 @@ class SynchronousExecutorTest(test.TestCase, _FuturesTestMixin): pass -@testtools.skipIf(not EVENTLET_AVAILABLE, 'eventlet is not available') +@testtools.skipIf(not eu.EVENTLET_AVAILABLE, 'eventlet is not available') class GreenThreadPoolExecutorTest(test.TestCase, _FuturesTestMixin): def _make_executor(self, max_workers): return futures.GreenThreadPoolExecutor(max_workers=max_workers) diff --git a/taskflow/tests/unit/test_retries.py b/taskflow/tests/unit/test_retries.py index 27a90d26..b459184b 100644 --- a/taskflow/tests/unit/test_retries.py +++ b/taskflow/tests/unit/test_retries.py @@ -27,7 +27,7 @@ from taskflow import test from taskflow.tests import utils from taskflow.types import failure from taskflow.types import futures -from taskflow.utils import async_utils as au +from taskflow.utils import eventlet_utils as eu class FailingRetry(retry.Retry): @@ -980,7 +980,7 @@ class ParallelEngineWithThreadsTest(RetryTest, max_workers=self._EXECUTOR_WORKERS) -@testtools.skipIf(not au.EVENTLET_AVAILABLE, 'eventlet is not available') +@testtools.skipIf(not eu.EVENTLET_AVAILABLE, 'eventlet is not available') class ParallelEngineWithEventletTest(RetryTest, test.TestCase): def _make_engine(self, flow, flow_detail=None, executor=None): diff --git a/taskflow/tests/unit/test_suspend.py b/taskflow/tests/unit/test_suspend.py index 08b2a83b..e5d0288f 100644 --- a/taskflow/tests/unit/test_suspend.py +++ b/taskflow/tests/unit/test_suspend.py @@ -23,7 +23,7 @@ from taskflow import states from taskflow import test from taskflow.tests import utils from taskflow.types import futures -from taskflow.utils import async_utils as au +from taskflow.utils import eventlet_utils as eu class SuspendingListener(utils.CaptureListener): @@ -212,7 +212,7 @@ class ParallelEngineWithThreadsTest(SuspendTest, test.TestCase): max_workers=self._EXECUTOR_WORKERS) -@testtools.skipIf(not au.EVENTLET_AVAILABLE, 'eventlet is not available') +@testtools.skipIf(not eu.EVENTLET_AVAILABLE, 'eventlet is not available') class ParallelEngineWithEventletTest(SuspendTest, test.TestCase): def _make_engine(self, flow, flow_detail=None, executor=None): diff --git a/taskflow/tests/unit/test_utils_async_utils.py b/taskflow/tests/unit/test_utils_async_utils.py index 7bb033b8..b538c2ee 100644 --- a/taskflow/tests/unit/test_utils_async_utils.py +++ b/taskflow/tests/unit/test_utils_async_utils.py @@ -19,6 +19,7 @@ import testtools from taskflow import test from taskflow.types import futures from taskflow.utils import async_utils as au +from taskflow.utils import eventlet_utils as eu class WaitForAnyTestsMixin(object): @@ -52,7 +53,7 @@ class WaitForAnyTestsMixin(object): self.assertIs(done.pop(), f2) -@testtools.skipIf(not au.EVENTLET_AVAILABLE, 'eventlet is not available') +@testtools.skipIf(not eu.EVENTLET_AVAILABLE, 'eventlet is not available') class AsyncUtilsEventletTest(test.TestCase, WaitForAnyTestsMixin): def _make_executor(self, max_workers): diff --git a/taskflow/types/futures.py b/taskflow/types/futures.py index 8c3d5504..1c8c4938 100644 --- a/taskflow/types/futures.py +++ b/taskflow/types/futures.py @@ -27,11 +27,11 @@ try: from eventlet import greenpool from eventlet import patcher as greenpatcher from eventlet import queue as greenqueue - EVENTLET_AVAILABLE = True except ImportError: - EVENTLET_AVAILABLE = False + pass from taskflow.types import timing +from taskflow.utils import eventlet_utils as eu from taskflow.utils import threading_utils as tu @@ -245,7 +245,8 @@ class _GreenWorker(object): class GreenFuture(Future): def __init__(self): super(GreenFuture, self).__init__() - assert EVENTLET_AVAILABLE, 'eventlet is needed to use a green future' + eu.check_for_eventlet(RuntimeError('Eventlet is needed to use a green' + ' future')) # NOTE(harlowja): replace the built-in condition with a greenthread # compatible one so that when getting the result of this future the # functions will correctly yield to eventlet. If this is not done then @@ -266,7 +267,8 @@ class GreenThreadPoolExecutor(_futures.Executor): """ def __init__(self, max_workers=1000): - assert EVENTLET_AVAILABLE, 'eventlet is needed to use a green executor' + eu.check_for_eventlet(RuntimeError('Eventlet is needed to use a green' + ' executor')) if max_workers <= 0: raise ValueError("Max workers must be greater than zero") self._max_workers = max_workers diff --git a/taskflow/utils/async_utils.py b/taskflow/utils/async_utils.py index 2fa3b5f6..71eafa18 100644 --- a/taskflow/utils/async_utils.py +++ b/taskflow/utils/async_utils.py @@ -19,11 +19,11 @@ from concurrent.futures import _base try: from eventlet.green import threading as greenthreading - EVENTLET_AVAILABLE = True except ImportError: - EVENTLET_AVAILABLE = False + pass from taskflow.types import futures +from taskflow.utils import eventlet_utils as eu _DONE_STATES = frozenset([ @@ -94,7 +94,8 @@ def _partition_futures(fs): def _wait_for_any_green(fs, timeout=None): - assert EVENTLET_AVAILABLE, 'eventlet is needed to wait on green futures' + eu.check_for_eventlet(RuntimeError('Eventlet is needed to wait on' + ' green futures')) with _base._AcquireFutures(fs): done, not_done = _partition_futures(fs) diff --git a/taskflow/utils/eventlet_utils.py b/taskflow/utils/eventlet_utils.py new file mode 100644 index 00000000..68dd355a --- /dev/null +++ b/taskflow/utils/eventlet_utils.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +try: + import eventlet as _eventlet # noqa + EVENTLET_AVAILABLE = True +except ImportError: + EVENTLET_AVAILABLE = False + + +def check_for_eventlet(exc=None): + """Check if eventlet is available and if not raise a runtime error. + + :param exc: exception to raise instead of raising a runtime error + :type exc: exception + """ + if not EVENTLET_AVAILABLE: + if exc is None: + raise RuntimeError('Eventlet is not current available') + else: + raise exc From 426484fe1abf88dc9a1d587b3cc7fe0c688362da Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 16 Jan 2015 12:27:49 -0800 Subject: [PATCH 204/240] Mirror the task executor methods in the retry action To make understanding the retry action easier have it have similar execute and revert functions that the task executor/action has so that it's easier to understand the common pattern used for execution of the different atom types. Change-Id: Ia5bb1a6d9684d1add8a213dfb6165c2fea3f9d70 --- .../engines/action_engine/actions/retry.py | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/taskflow/engines/action_engine/actions/retry.py b/taskflow/engines/action_engine/actions/retry.py index 06a81fd4..bd96c899 100644 --- a/taskflow/engines/action_engine/actions/retry.py +++ b/taskflow/engines/action_engine/actions/retry.py @@ -25,6 +25,22 @@ from taskflow.types import futures LOG = logging.getLogger(__name__) +def _execute_retry(retry, arguments): + try: + result = retry.execute(**arguments) + except Exception: + result = failure.Failure() + return (ex.EXECUTED, result) + + +def _revert_retry(retry, arguments): + try: + result = retry.revert(**arguments) + except Exception: + result = failure.Failure() + return (ex.REVERTED, result) + + class RetryAction(base.Action): """An action that handles executing, state changes, ... of retry atoms.""" @@ -38,14 +54,14 @@ class RetryAction(base.Action): def _get_retry_args(self, retry, addons=None): scope_walker = self._walker_factory(retry) - kwargs = self._storage.fetch_mapped_args(retry.rebind, - atom_name=retry.name, - scope_walker=scope_walker) + arguments = self._storage.fetch_mapped_args(retry.rebind, + atom_name=retry.name, + scope_walker=scope_walker) history = self._storage.get_retry_history(retry.name) - kwargs[retry_atom.EXECUTE_REVERT_HISTORY] = history + arguments[retry_atom.EXECUTE_REVERT_HISTORY] = history if addons: - kwargs.update(addons) - return kwargs + arguments.update(addons) + return arguments def change_state(self, retry, state, result=base.NO_RESULT): old_state = self._storage.get_atom_state(retry.name) @@ -74,13 +90,6 @@ class RetryAction(base.Action): def execute(self, retry): - def _execute_retry(kwargs): - try: - result = retry.execute(**kwargs) - except Exception: - result = failure.Failure() - return (ex.EXECUTED, result) - def _on_done_callback(fut): result = fut.result()[-1] if isinstance(result, failure.Failure): @@ -89,7 +98,7 @@ class RetryAction(base.Action): self.change_state(retry, states.SUCCESS, result=result) self.change_state(retry, states.RUNNING) - fut = self._executor.submit(_execute_retry, + fut = self._executor.submit(_execute_retry, retry, self._get_retry_args(retry)) fut.add_done_callback(_on_done_callback) fut.atom = retry @@ -97,13 +106,6 @@ class RetryAction(base.Action): def revert(self, retry): - def _execute_retry(kwargs): - try: - result = retry.revert(**kwargs) - except Exception: - result = failure.Failure() - return (ex.REVERTED, result) - def _on_done_callback(fut): result = fut.result()[-1] if isinstance(result, failure.Failure): @@ -115,7 +117,7 @@ class RetryAction(base.Action): arg_addons = { retry_atom.REVERT_FLOW_FAILURES: self._storage.get_failures(), } - fut = self._executor.submit(_execute_retry, + fut = self._executor.submit(_revert_retry, retry, self._get_retry_args(retry, addons=arg_addons)) fut.add_done_callback(_on_done_callback) @@ -124,5 +126,5 @@ class RetryAction(base.Action): def on_failure(self, retry, atom, last_failure): self._storage.save_retry_failure(retry.name, atom.name, last_failure) - kwargs = self._get_retry_args(retry) - return retry.on_failure(**kwargs) + arguments = self._get_retry_args(retry) + return retry.on_failure(**arguments) From ac5345e651af79f52d16a92874afa23c047a3e52 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 16 Jan 2015 13:09:28 -0800 Subject: [PATCH 205/240] Have the serial task executor shutdown/restart its executor When the serial task executor is stopped, have it match what the other executors do and shutdown its executor, and on start have it restart/start it to be come functional again. This matches what the other executors do and makes it easier to understand the common pattern that is applied/used. Change-Id: Id62b588b05262aa9e334a64f53e4c4a0d5b66159 --- taskflow/engines/action_engine/executor.py | 6 ++++++ taskflow/types/futures.py | 13 +++++++++++++ 2 files changed, 19 insertions(+) diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index 1c925678..d0d71e56 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -359,6 +359,12 @@ class SerialTaskExecutor(TaskExecutor): def __init__(self): self._executor = futures.SynchronousExecutor() + def start(self): + self._executor.restart() + + def stop(self): + self._executor.shutdown() + def execute_task(self, task, task_uuid, arguments, progress_callback=None): fut = self._executor.submit(_execute_task, task, arguments, diff --git a/taskflow/types/futures.py b/taskflow/types/futures.py index 8c3d5504..0248d996 100644 --- a/taskflow/types/futures.py +++ b/taskflow/types/futures.py @@ -51,6 +51,10 @@ class _Gatherer(object): def statistics(self): return self._stats + def clear(self): + with self._stats_lock: + self._stats = ExecutorStatistics() + def _capture_stats(self, watch, fut): watch.stop() with self._stats_lock: @@ -197,6 +201,15 @@ class SynchronousExecutor(_futures.Executor): def shutdown(self, wait=True): self._shutoff = True + def restart(self): + """Restarts this executor (*iff* previously shutoff/shutdown). + + NOTE(harlowja): clears any previously gathered statistics. + """ + if self._shutoff: + self._shutoff = False + self._gatherer.clear() + @property def statistics(self): """:class:`.ExecutorStatistics` about the executors executions.""" From f14ee9ea5cf2f42951a1322a1250c28f3e447585 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 19 Dec 2014 16:34:21 -0800 Subject: [PATCH 206/240] Move the jobboard/job bases to a jobboard/base module In order to match the directory/module layout of the other pluggable backends better move the jobboard modules that define the base abstract classes into a single base file. This makes it easier to look at the taskflow code-base and understand the common layout. This also makes the docs for the zookeeper jobboard better and includes them in the generated developer docs under a implementations section. Change-Id: I36f29c37dcf2403782a75e45665bd7c0a146a06e --- doc/source/jobs.rst | 19 ++- taskflow/jobs/backends/impl_zookeeper.py | 46 ++++++-- taskflow/jobs/{jobboard.py => base.py} | 91 ++++++++++++++ taskflow/jobs/job.py | 111 ------------------ .../tests/unit/conductor/test_conductor.py | 6 +- 5 files changed, 144 insertions(+), 129 deletions(-) rename taskflow/jobs/{jobboard.py => base.py} (74%) delete mode 100644 taskflow/jobs/job.py diff --git a/doc/source/jobs.rst b/doc/source/jobs.rst index f36d69a1..06f1123e 100644 --- a/doc/source/jobs.rst +++ b/doc/source/jobs.rst @@ -28,14 +28,14 @@ Definitions =========== Jobs - A :py:class:`job ` consists of a unique identifier, + A :py:class:`job ` consists of a unique identifier, name, and a reference to a :py:class:`logbook ` which contains the details of the work that has been or should be/will be completed to finish the work that has been created for that job. Jobboards - A :py:class:`jobboard ` is responsible for + A :py:class:`jobboard ` is responsible for managing the posting, ownership, and delivery of jobs. It acts as the location where jobs can be posted, claimed and searched for; typically by iteration or notification. Jobboards may be backed by different *capable* @@ -202,6 +202,11 @@ Additional *configuration* parameters: when your program uses eventlet and you want to instruct kazoo to use an eventlet compatible handler (such as the `eventlet handler`_). +.. note:: + + See :py:class:`~taskflow.jobs.backends.impl_zookeeper.ZookeeperJobBoard` + for implementation details. + Considerations ============== @@ -254,15 +259,19 @@ the claim by then, therefore both would be *working* on a job. Interfaces ========== +.. automodule:: taskflow.jobs.base .. automodule:: taskflow.jobs.backends -.. automodule:: taskflow.jobs.job -.. automodule:: taskflow.jobs.jobboard + +Implementations +=============== + +.. automodule:: taskflow.jobs.backends.impl_zookeeper Hierarchy ========= .. inheritance-diagram:: - taskflow.jobs.jobboard + taskflow.jobs.base taskflow.jobs.backends.impl_zookeeper :parts: 1 diff --git a/taskflow/jobs/backends/impl_zookeeper.py b/taskflow/jobs/backends/impl_zookeeper.py index 186be8e6..36e0daea 100644 --- a/taskflow/jobs/backends/impl_zookeeper.py +++ b/taskflow/jobs/backends/impl_zookeeper.py @@ -29,8 +29,7 @@ from oslo_utils import uuidutils import six from taskflow import exceptions as excp -from taskflow.jobs import job as base_job -from taskflow.jobs import jobboard +from taskflow.jobs import base from taskflow import logging from taskflow import states from taskflow.types import timing as tt @@ -62,7 +61,9 @@ def _check_who(who): raise ValueError("Job applicant must be non-empty") -class ZookeeperJob(base_job.Job): +class ZookeeperJob(base.Job): + """A zookeeper job.""" + def __init__(self, name, board, client, backend, path, uuid=None, details=None, book=None, book_data=None, created_on=None): @@ -95,10 +96,12 @@ class ZookeeperJob(base_job.Job): @property def sequence(self): + """Sequence number of the current job.""" return self._sequence @property def root(self): + """The parent path of the job in zookeeper.""" return self._root def _get_node_attr(self, path, attr_name, trans_func=None): @@ -232,14 +235,14 @@ class ZookeeperJob(base_job.Job): class ZookeeperJobBoardIterator(six.Iterator): - """Iterator over a zookeeper jobboard. + """Iterator over a zookeeper jobboard that iterates over potential jobs. It supports the following attributes/constructor arguments: - * ensure_fresh: boolean that requests that during every fetch of a new + * ``ensure_fresh``: boolean that requests that during every fetch of a new set of jobs this will cause the iterator to force the backend to refresh (ensuring that the jobboard has the most recent job listings). - * only_unclaimed: boolean that indicates whether to only iterate + * ``only_unclaimed``: boolean that indicates whether to only iterate over unclaimed jobs. """ @@ -288,7 +291,30 @@ class ZookeeperJobBoardIterator(six.Iterator): return job -class ZookeeperJobBoard(jobboard.NotifyingJobBoard): +class ZookeeperJobBoard(base.NotifyingJobBoard): + """A jobboard backend by zookeeper. + + Powered by the `kazoo `_ library. + + This jobboard creates *sequenced* persistent znodes in a directory in + zookeeper (that directory defaults ``/taskflow/jobs``) and uses zookeeper + watches to notify other jobboards that the job which was posted using the + :meth:`.post` method (this creates a znode with contents/details in json) + The users of those jobboard(s) (potentially on disjoint sets of machines) + can then iterate over the available jobs and decide if they want to attempt + to claim one of the jobs they have iterated over. If so they will then + attempt to contact zookeeper and will attempt to create a ephemeral znode + using the name of the persistent znode + ".lock" as a postfix. If the + entity trying to use the jobboard to :meth:`.claim` the job is able to + create a ephemeral znode with that name then it will be allowed (and + expected) to perform whatever *work* the contents of that job that it + locked described. Once finished the ephemeral znode and persistent znode + may be deleted (if successfully completed) in a single transcation or if + not successfull (or the entity that claimed the znode dies) the ephemeral + znode will be released (either manually by using :meth:`.abandon` or + automatically by zookeeper the ephemeral is deemed to be lost). + """ + def __init__(self, name, conf, client=None, persistence=None, emit_notifications=True): super(ZookeeperJobBoard, self).__init__(name, conf) @@ -373,7 +399,7 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): job = self._known_jobs.pop(path, None) if job is not None: LOG.debug("Removed job that was at path '%s'", path) - self._emit(jobboard.REMOVAL, details={'job': job}) + self._emit(base.REMOVAL, details={'job': job}) def _process_child(self, path, request): """Receives the result of a child data fetch request.""" @@ -412,7 +438,7 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): self._known_jobs[path] = job self._job_cond.notify_all() if job is not None: - self._emit(jobboard.POSTED, details={'job': job}) + self._emit(base.POSTED, details={'job': job}) def _on_job_posting(self, children, delayed=True): LOG.debug("Got children %s under path %s", children, self.path) @@ -498,7 +524,7 @@ class ZookeeperJobBoard(jobboard.NotifyingJobBoard): with self._job_cond: self._known_jobs[job_path] = job self._job_cond.notify_all() - self._emit(jobboard.POSTED, details={'job': job}) + self._emit(base.POSTED, details={'job': job}) return job def claim(self, job, who): diff --git a/taskflow/jobs/jobboard.py b/taskflow/jobs/base.py similarity index 74% rename from taskflow/jobs/jobboard.py rename to taskflow/jobs/base.py index 0938d0e7..eea5b12b 100644 --- a/taskflow/jobs/jobboard.py +++ b/taskflow/jobs/base.py @@ -17,11 +17,102 @@ import abc +from oslo_utils import uuidutils import six from taskflow.types import notifier +@six.add_metaclass(abc.ABCMeta) +class Job(object): + """A abstraction that represents a named and trackable unit of work. + + A job connects a logbook, a owner, last modified and created on dates and + any associated state that the job has. Since it is a connector to a + logbook, which are each associated with a set of factories that can create + set of flows, it is the current top-level container for a piece of work + that can be owned by an entity (typically that entity will read those + logbooks and run any contained flows). + + Only one entity will be allowed to own and operate on the flows contained + in a job at a given time (for the foreseeable future). + + NOTE(harlowja): It is the object that will be transferred to another + entity on failure so that the contained flows ownership can be + transferred to the secondary entity/owner for resumption, continuation, + reverting... + """ + + def __init__(self, name, uuid=None, details=None): + if uuid: + self._uuid = uuid + else: + self._uuid = uuidutils.generate_uuid() + self._name = name + if not details: + details = {} + self._details = details + + @abc.abstractproperty + def last_modified(self): + """The datetime the job was last modified.""" + pass + + @abc.abstractproperty + def created_on(self): + """The datetime the job was created on.""" + pass + + @abc.abstractproperty + def board(self): + """The board this job was posted on or was created from.""" + + @abc.abstractproperty + def state(self): + """The current state of this job.""" + + @abc.abstractproperty + def book(self): + """Logbook associated with this job. + + If no logbook is associated with this job, this property is None. + """ + + @abc.abstractproperty + def book_uuid(self): + """UUID of logbook associated with this job. + + If no logbook is associated with this job, this property is None. + """ + + @abc.abstractproperty + def book_name(self): + """Name of logbook associated with this job. + + If no logbook is associated with this job, this property is None. + """ + + @property + def uuid(self): + """The uuid of this job.""" + return self._uuid + + @property + def details(self): + """A dictionary of any details associated with this job.""" + return self._details + + @property + def name(self): + """The non-uniquely identifying name of this job.""" + return self._name + + def __str__(self): + """Pretty formats the job into something *more* meaningful.""" + return "%s %s (%s): %s" % (type(self).__name__, + self.name, self.uuid, self.details) + + @six.add_metaclass(abc.ABCMeta) class JobBoard(object): """A place where jobs can be posted, reposted, claimed and transferred. diff --git a/taskflow/jobs/job.py b/taskflow/jobs/job.py deleted file mode 100644 index a7dd04f7..00000000 --- a/taskflow/jobs/job.py +++ /dev/null @@ -1,111 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (C) 2013 Rackspace Hosting Inc. All Rights Reserved. -# Copyright (C) 2013 Yahoo! Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc - -from oslo_utils import uuidutils -import six - - -@six.add_metaclass(abc.ABCMeta) -class Job(object): - """A abstraction that represents a named and trackable unit of work. - - A job connects a logbook, a owner, last modified and created on dates and - any associated state that the job has. Since it is a connector to a - logbook, which are each associated with a set of factories that can create - set of flows, it is the current top-level container for a piece of work - that can be owned by an entity (typically that entity will read those - logbooks and run any contained flows). - - Only one entity will be allowed to own and operate on the flows contained - in a job at a given time (for the foreseeable future). - - NOTE(harlowja): It is the object that will be transferred to another - entity on failure so that the contained flows ownership can be - transferred to the secondary entity/owner for resumption, continuation, - reverting... - """ - - def __init__(self, name, uuid=None, details=None): - if uuid: - self._uuid = uuid - else: - self._uuid = uuidutils.generate_uuid() - self._name = name - if not details: - details = {} - self._details = details - - @abc.abstractproperty - def last_modified(self): - """The datetime the job was last modified.""" - pass - - @abc.abstractproperty - def created_on(self): - """The datetime the job was created on.""" - pass - - @abc.abstractproperty - def board(self): - """The board this job was posted on or was created from.""" - - @abc.abstractproperty - def state(self): - """The current state of this job.""" - - @abc.abstractproperty - def book(self): - """Logbook associated with this job. - - If no logbook is associated with this job, this property is None. - """ - - @abc.abstractproperty - def book_uuid(self): - """UUID of logbook associated with this job. - - If no logbook is associated with this job, this property is None. - """ - - @abc.abstractproperty - def book_name(self): - """Name of logbook associated with this job. - - If no logbook is associated with this job, this property is None. - """ - - @property - def uuid(self): - """The uuid of this job.""" - return self._uuid - - @property - def details(self): - """A dictionary of any details associated with this job.""" - return self._details - - @property - def name(self): - """The non-uniquely identifying name of this job.""" - return self._name - - def __str__(self): - """Pretty formats the job into something *more* meaningful.""" - return "%s %s (%s): %s" % (type(self).__name__, - self.name, self.uuid, self.details) diff --git a/taskflow/tests/unit/conductor/test_conductor.py b/taskflow/tests/unit/conductor/test_conductor.py index b861c12b..137d0f3a 100644 --- a/taskflow/tests/unit/conductor/test_conductor.py +++ b/taskflow/tests/unit/conductor/test_conductor.py @@ -22,7 +22,7 @@ from zake import fake_client from taskflow.conductors import single_threaded as stc from taskflow import engines from taskflow.jobs.backends import impl_zookeeper -from taskflow.jobs import jobboard +from taskflow.jobs import base from taskflow.patterns import linear_flow as lf from taskflow.persistence.backends import impl_memory from taskflow import states as st @@ -93,7 +93,7 @@ class SingleThreadedConductorTest(test_utils.EngineTestBase, test.TestCase): def on_consume(state, details): consumed_event.set() - components.board.notifier.register(jobboard.REMOVAL, on_consume) + components.board.notifier.register(base.REMOVAL, on_consume) with close_many(components.conductor, components.client): t = threading_utils.daemon_thread(components.conductor.run) t.start() @@ -122,7 +122,7 @@ class SingleThreadedConductorTest(test_utils.EngineTestBase, test.TestCase): def on_consume(state, details): consumed_event.set() - components.board.notifier.register(jobboard.REMOVAL, on_consume) + components.board.notifier.register(base.REMOVAL, on_consume) with close_many(components.conductor, components.client): t = threading_utils.daemon_thread(components.conductor.run) t.start() From bb384577bcf1ffd0c876fcb174a80ea0a120b52f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 19 Dec 2014 19:26:12 -0800 Subject: [PATCH 207/240] Move implementation(s) to there own sections Instead of putting implementation(s) under the interfaces section put the implementation(s) under there own section. This also includes some other tweaks to refer to those implementation(s) where appropriate. Change-Id: Iffdc0439c843e7f70cf873e5a75501feb51f96c7 --- doc/source/conductors.rst | 4 +++ doc/source/engines.rst | 6 +++- doc/source/persistence.rst | 33 +++++++++++++++++-- taskflow/persistence/backends/impl_dir.py | 8 ++--- .../persistence/backends/impl_sqlalchemy.py | 8 ++--- .../persistence/backends/impl_zookeeper.py | 10 +++--- 6 files changed, 53 insertions(+), 16 deletions(-) diff --git a/doc/source/conductors.rst b/doc/source/conductors.rst index 25eb75c8..56fb0e0e 100644 --- a/doc/source/conductors.rst +++ b/doc/source/conductors.rst @@ -63,6 +63,10 @@ Interfaces ========== .. automodule:: taskflow.conductors.base + +Implementations +=============== + .. automodule:: taskflow.conductors.single_threaded Hierarchy diff --git a/doc/source/engines.rst b/doc/source/engines.rst index 04d462a4..0231ae86 100644 --- a/doc/source/engines.rst +++ b/doc/source/engines.rst @@ -330,6 +330,11 @@ saved for this execution. Interfaces ========== +.. automodule:: taskflow.engines.base + +Implementations +=============== + .. automodule:: taskflow.engines.action_engine.analyzer .. automodule:: taskflow.engines.action_engine.compiler .. automodule:: taskflow.engines.action_engine.completer @@ -339,7 +344,6 @@ Interfaces .. automodule:: taskflow.engines.action_engine.runtime .. automodule:: taskflow.engines.action_engine.scheduler .. automodule:: taskflow.engines.action_engine.scopes -.. automodule:: taskflow.engines.base Hierarchy ========= diff --git a/doc/source/persistence.rst b/doc/source/persistence.rst index 0da68de1..3ea8c86d 100644 --- a/doc/source/persistence.rst +++ b/doc/source/persistence.rst @@ -155,6 +155,11 @@ Memory Retains all data in local memory (not persisted to reliable storage). Useful for scenarios where persistence is not required (and also in unit tests). +.. note:: + + See :py:class:`~taskflow.persistence.backends.impl_memory.MemoryBackend` + for implementation details. + Files ----- @@ -166,6 +171,11 @@ from the same local machine only). Useful for cases where a *more* reliable persistence is desired along with the simplicity of files and directories (a concept everyone is familiar with). +.. note:: + + See :py:class:`~taskflow.persistence.backends.impl_dir.DirBackend` + for implementation details. + Sqlalchemy ---------- @@ -228,6 +238,11 @@ parent_uuid VARCHAR False .. _sqlalchemy: http://www.sqlalchemy.org/docs/ .. _ACID: https://en.wikipedia.org/wiki/ACID +.. note:: + + See :py:class:`~taskflow.persistence.backends.impl_sqlalchemy.SQLAlchemyBackend` + for implementation details. + Zookeeper --------- @@ -241,6 +256,11 @@ logbook represented as znodes. Since zookeeper is also distributed it is also able to resume a engine from a peer machine (having similar functionality as the database connection types listed previously). +.. note:: + + See :py:class:`~taskflow.persistence.backends.impl_zookeeper.ZkBackend` + for implementation details. + .. _zookeeper: http://zookeeper.apache.org .. _kazoo: http://kazoo.readthedocs.org/ @@ -251,12 +271,21 @@ Interfaces .. automodule:: taskflow.persistence.backends.base .. automodule:: taskflow.persistence.logbook +Implementations +=============== + +.. automodule:: taskflow.persistence.backends.impl_dir +.. automodule:: taskflow.persistence.backends.impl_memory +.. automodule:: taskflow.persistence.backends.impl_sqlalchemy +.. automodule:: taskflow.persistence.backends.impl_zookeeper + Hierarchy ========= .. inheritance-diagram:: - taskflow.persistence.backends.impl_memory - taskflow.persistence.backends.impl_zookeeper + taskflow.persistence.backends.base taskflow.persistence.backends.impl_dir + taskflow.persistence.backends.impl_memory taskflow.persistence.backends.impl_sqlalchemy + taskflow.persistence.backends.impl_zookeeper :parts: 2 diff --git a/taskflow/persistence/backends/impl_dir.py b/taskflow/persistence/backends/impl_dir.py index 0a687473..6b6a0582 100644 --- a/taskflow/persistence/backends/impl_dir.py +++ b/taskflow/persistence/backends/impl_dir.py @@ -46,11 +46,11 @@ class DirBackend(base.Backend): guarantee that there will be no interprocess race conditions when writing and reading by using a consistent hierarchy of file based locks. - Example conf: + Example configuration:: - conf = { - "path": "/tmp/taskflow", - } + conf = { + "path": "/tmp/taskflow", + } """ def __init__(self, conf): super(DirBackend, self).__init__(conf) diff --git a/taskflow/persistence/backends/impl_sqlalchemy.py b/taskflow/persistence/backends/impl_sqlalchemy.py index d84b9b27..41f29c9c 100644 --- a/taskflow/persistence/backends/impl_sqlalchemy.py +++ b/taskflow/persistence/backends/impl_sqlalchemy.py @@ -183,11 +183,11 @@ def _ping_listener(dbapi_conn, connection_rec, connection_proxy): class SQLAlchemyBackend(base.Backend): """A sqlalchemy backend. - Example conf: + Example configuration:: - conf = { - "connection": "sqlite:////tmp/test.db", - } + conf = { + "connection": "sqlite:////tmp/test.db", + } """ def __init__(self, conf, engine=None): super(SQLAlchemyBackend, self).__init__(conf) diff --git a/taskflow/persistence/backends/impl_zookeeper.py b/taskflow/persistence/backends/impl_zookeeper.py index ca801b43..c2a07b8e 100644 --- a/taskflow/persistence/backends/impl_zookeeper.py +++ b/taskflow/persistence/backends/impl_zookeeper.py @@ -43,12 +43,12 @@ class ZkBackend(base.Backend): inside those directories that represent the contents of those objects for later reading and writing. - Example conf: + Example configuration:: - conf = { - "hosts": "192.168.0.1:2181,192.168.0.2:2181,192.168.0.3:2181", - "path": "/taskflow", - } + conf = { + "hosts": "192.168.0.1:2181,192.168.0.2:2181,192.168.0.3:2181", + "path": "/taskflow", + } """ def __init__(self, conf, client=None): super(ZkBackend, self).__init__(conf) From cb27080ea3cd5cddd7f91d866f6a9d1214c9e885 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 16 Jan 2015 15:37:44 -0800 Subject: [PATCH 208/240] Increase robustness of WBE producer/consumers Use the kombu provided ensure() decorator/wrapper along with sensible default settings to ensure that retries are attempted when kombu detects recoverable connection or recoverable channel errors have occurred. Change-Id: If47f72d02561d0b5d556ac386869a6122c8b871d --- taskflow/engines/worker_based/engine.py | 5 +- taskflow/engines/worker_based/proxy.py | 103 ++++++++++++++---- taskflow/engines/worker_based/worker.py | 2 + .../tests/unit/worker_based/test_creation.py | 9 +- .../tests/unit/worker_based/test_proxy.py | 66 +++++++---- 5 files changed, 140 insertions(+), 45 deletions(-) diff --git a/taskflow/engines/worker_based/engine.py b/taskflow/engines/worker_based/engine.py index 8011222c..a161ee58 100644 --- a/taskflow/engines/worker_based/engine.py +++ b/taskflow/engines/worker_based/engine.py @@ -23,7 +23,7 @@ from taskflow import storage as t_storage class WorkerBasedActionEngine(engine.ActionEngine): """Worker based action engine. - Specific backend options: + Specific backend options (extracted from provided engine options): :param exchange: broker exchange exchange name in which executor / worker communication is performed @@ -40,6 +40,8 @@ class WorkerBasedActionEngine(engine.ActionEngine): for will have its result become a `RequestTimeout` exception instead of its normally returned value (or raised exception). + :param retry_options: retry specific options (used to configure how kombu + handles retrying under tolerable/transient failures). """ _storage_factory = t_storage.SingleThreadedStorage @@ -66,6 +68,7 @@ class WorkerBasedActionEngine(engine.ActionEngine): uuid=flow_detail.uuid, url=options.get('url'), exchange=options.get('exchange', 'default'), + retry_options=options.get('retry_options'), topics=options.get('topics', []), transport=options.get('transport'), transport_options=options.get('transport_options'), diff --git a/taskflow/engines/worker_based/proxy.py b/taskflow/engines/worker_based/proxy.py index 1c870595..9a3b8e09 100644 --- a/taskflow/engines/worker_based/proxy.py +++ b/taskflow/engines/worker_based/proxy.py @@ -44,6 +44,26 @@ _TransportDetails = collections.namedtuple('_TransportDetails', class Proxy(object): """A proxy processes messages from/to the named exchange.""" + # Settings that are by default used for consumers/producers to reconnect + # under tolerable/transient failures... + # + # See: http://kombu.readthedocs.org/en/latest/reference/kombu.html for + # what these values imply... + _DEFAULT_RETRY_OPTIONS = { + # The number of seconds we start sleeping for. + 'interval_start': 1, + # How many seconds added to the interval for each retry. + 'interval_step': 1, + # Maximum number of seconds to sleep between each retry. + 'interval_max': 1, + # Maximum number of times to retry. + 'max_retries': 3, + } + # This is the only provided option that should be an int, the others + # are allowed to be floats; used when we check that the user-provided + # value is valid... + _RETRY_INT_OPTS = frozenset(['max_retries']) + def __init__(self, topic, exchange_name, type_handlers, on_wait=None, **kwargs): self._topic = topic @@ -56,9 +76,28 @@ class Proxy(object): # running, otherwise requeue them. lambda data, message: not self.is_running) + # TODO(harlowja): make these keyword arguments explict... url = kwargs.get('url') transport = kwargs.get('transport') transport_opts = kwargs.get('transport_options') + ensure_options = self._DEFAULT_RETRY_OPTIONS.copy() + if 'retry_options' in kwargs and kwargs['retry_options'] is not None: + # Override the defaults with any user provided values... + usr_retry_options = kwargs['retry_options'] + for k in set(six.iterkeys(ensure_options)): + if k in usr_retry_options: + # Ensure that the right type is passed in... + val = usr_retry_options[k] + if k in self._RETRY_INT_OPTS: + tmp_val = int(val) + else: + tmp_val = float(val) + if tmp_val < 0: + raise ValueError("Expected value greater or equal to" + " zero for 'retry_options' %s; got" + " %s instead" % (k, val)) + ensure_options[k] = tmp_val + self._ensure_options = ensure_options self._drain_events_timeout = DRAIN_EVENTS_PERIOD if transport == 'memory' and transport_opts: @@ -113,34 +152,60 @@ class Proxy(object): routing_keys = [routing_key] else: routing_keys = routing_key + + def _publish(producer, routing_key): + queue = self._make_queue(routing_key, self._exchange) + producer.publish(body=msg.to_dict(), + routing_key=routing_key, + exchange=self._exchange, + declare=[queue], + type=msg.TYPE, + reply_to=reply_to, + correlation_id=correlation_id) + + def _publish_errback(exc, interval): + LOG.exception('Publishing error: %s', exc) + LOG.info('Retry triggering in %s seconds', interval) + LOG.debug("Sending '%s' using routing keys %s", msg, routing_keys) - with kombu.producers[self._conn].acquire(block=True) as producer: - for routing_key in routing_keys: - queue = self._make_queue(routing_key, self._exchange) - producer.publish(body=msg.to_dict(), - routing_key=routing_key, - exchange=self._exchange, - declare=[queue], - type=msg.TYPE, - reply_to=reply_to, - correlation_id=correlation_id) + with kombu.connections[self._conn].acquire(block=True) as conn: + with conn.Producer() as producer: + ensure_kwargs = self._ensure_options.copy() + ensure_kwargs['errback'] = _publish_errback + safe_publish = conn.ensure(producer, _publish, **ensure_kwargs) + for routing_key in routing_keys: + safe_publish(producer, routing_key) def start(self): """Start proxy.""" + + def _drain(conn, timeout): + try: + conn.drain_events(timeout=timeout) + except socket.timeout: + pass + + def _drain_errback(exc, interval): + LOG.exception('Draining error: %s', exc) + LOG.info('Retry triggering in %s seconds', interval) + LOG.info("Starting to consume from the '%s' exchange.", self._exchange_name) with kombu.connections[self._conn].acquire(block=True) as conn: queue = self._make_queue(self._topic, self._exchange, channel=conn) - with conn.Consumer(queues=queue, - callbacks=[self._dispatcher.on_message]): + callbacks = [self._dispatcher.on_message] + with conn.Consumer(queues=queue, callbacks=callbacks) as consumer: + ensure_kwargs = self._ensure_options.copy() + ensure_kwargs['errback'] = _drain_errback + safe_drain = conn.ensure(consumer, _drain, **ensure_kwargs) self._running.set() - while self.is_running: - try: - conn.drain_events(timeout=self._drain_events_timeout) - except socket.timeout: - pass - if self._on_wait is not None: - self._on_wait() + try: + while self._running.is_set(): + safe_drain(conn, self._drain_events_timeout) + if self._on_wait is not None: + self._on_wait() + finally: + self._running.clear() def wait(self): """Wait until proxy is started.""" diff --git a/taskflow/engines/worker_based/worker.py b/taskflow/engines/worker_based/worker.py index 98e690ef..5ac0cf4f 100644 --- a/taskflow/engines/worker_based/worker.py +++ b/taskflow/engines/worker_based/worker.py @@ -80,6 +80,8 @@ class Worker(object): :param threads_count: threads count to be passed to the default executor :param transport: transport to be used (e.g. amqp, memory, etc.) :param transport_options: transport specific options + :param retry_options: retry specific options (used to configure how kombu + handles retrying under tolerable/transient failures). """ def __init__(self, exchange, topic, tasks, executor=None, **kwargs): diff --git a/taskflow/tests/unit/worker_based/test_creation.py b/taskflow/tests/unit/worker_based/test_creation.py index 6764926a..887498ce 100644 --- a/taskflow/tests/unit/worker_based/test_creation.py +++ b/taskflow/tests/unit/worker_based/test_creation.py @@ -49,7 +49,8 @@ class TestWorkerBasedActionEngine(test.MockTestCase): topics=[], transport=None, transport_options=None, - transition_timeout=mock.ANY) + transition_timeout=mock.ANY, + retry_options=None) ] self.assertEqual(self.master_mock.mock_calls, expected_calls) @@ -64,7 +65,8 @@ class TestWorkerBasedActionEngine(test.MockTestCase): transport='memory', transport_options={}, transition_timeout=200, - topics=topics) + topics=topics, + retry_options={}) expected_calls = [ mock.call.executor_class(uuid=eng.storage.flow_uuid, url=broker_url, @@ -72,7 +74,8 @@ class TestWorkerBasedActionEngine(test.MockTestCase): topics=topics, transport='memory', transport_options={}, - transition_timeout=200) + transition_timeout=200, + retry_options={}) ] self.assertEqual(self.master_mock.mock_calls, expected_calls) diff --git a/taskflow/tests/unit/worker_based/test_proxy.py b/taskflow/tests/unit/worker_based/test_proxy.py index a3c7d13f..daf9b60e 100644 --- a/taskflow/tests/unit/worker_based/test_proxy.py +++ b/taskflow/tests/unit/worker_based/test_proxy.py @@ -43,8 +43,11 @@ class TestProxy(test.MockTestCase): proxy.kombu, 'Producer') # connection mocking + def _ensure(obj, func, *args, **kwargs): + return func self.conn_inst_mock.drain_events.side_effect = [ socket.timeout, socket.timeout, KeyboardInterrupt] + self.conn_inst_mock.ensure = mock.MagicMock(side_effect=_ensure) # connections mocking self.connections_mock = self.patch( @@ -54,11 +57,8 @@ class TestProxy(test.MockTestCase): self.conn_inst_mock # producers mocking - self.producers_mock = self.patch( - "taskflow.engines.worker_based.proxy.kombu.producers", - attach_as='producers') - self.producers_mock.__getitem__().acquire().__enter__.return_value =\ - self.producer_inst_mock + self.conn_inst_mock.Producer.return_value.__enter__ = mock.MagicMock() + self.conn_inst_mock.Producer.return_value.__exit__ = mock.MagicMock() # consumer mocking self.conn_inst_mock.Consumer.return_value.__enter__ = mock.MagicMock() @@ -85,11 +85,38 @@ class TestProxy(test.MockTestCase): mock.call.connection.Consumer(queues=self.queue_inst_mock, callbacks=[mock.ANY]), mock.call.connection.Consumer().__enter__(), + mock.call.connection.ensure(mock.ANY, mock.ANY, + interval_start=mock.ANY, + interval_max=mock.ANY, + max_retries=mock.ANY, + interval_step=mock.ANY, + errback=mock.ANY), ] + calls + [ mock.call.connection.Consumer().__exit__(exc_type, mock.ANY, mock.ANY) ] + def proxy_publish_calls(self, calls, routing_key, exc_type=mock.ANY): + return [ + mock.call.connection.Producer(), + mock.call.connection.Producer().__enter__(), + mock.call.connection.ensure(mock.ANY, mock.ANY, + interval_start=mock.ANY, + interval_max=mock.ANY, + max_retries=mock.ANY, + interval_step=mock.ANY, + errback=mock.ANY), + mock.call.Queue(name=self._queue_name(routing_key), + routing_key=routing_key, + exchange=self.exchange_inst_mock, + durable=False, + auto_delete=True, + channel=None), + ] + calls + [ + mock.call.connection.Producer().__exit__(exc_type, mock.ANY, + mock.ANY) + ] + def proxy(self, reset_master_mock=False, **kwargs): proxy_kwargs = dict(topic=self.topic, exchange_name=self.exchange_name, @@ -133,24 +160,19 @@ class TestProxy(test.MockTestCase): routing_key = 'routing-key' task_uuid = 'task-uuid' - self.proxy(reset_master_mock=True).publish( - msg_mock, routing_key, correlation_id=task_uuid) + p = self.proxy(reset_master_mock=True) + p.publish(msg_mock, routing_key, correlation_id=task_uuid) - master_mock_calls = [ - mock.call.Queue(name=self._queue_name(routing_key), - exchange=self.exchange_inst_mock, - routing_key=routing_key, - durable=False, - auto_delete=True, - channel=None), - mock.call.producer.publish(body=msg_data, - routing_key=routing_key, - exchange=self.exchange_inst_mock, - correlation_id=task_uuid, - declare=[self.queue_inst_mock], - type=msg_mock.TYPE, - reply_to=None) - ] + mock_producer = mock.call.connection.Producer() + master_mock_calls = self.proxy_publish_calls([ + mock_producer.__enter__().publish(body=msg_data, + routing_key=routing_key, + exchange=self.exchange_inst_mock, + correlation_id=task_uuid, + declare=[self.queue_inst_mock], + type=msg_mock.TYPE, + reply_to=None) + ], routing_key) self.master_mock.assert_has_calls(master_mock_calls) def test_start(self): From 410efa738696dd998723dfe8f79aab3b3d7e6d2e Mon Sep 17 00:00:00 2001 From: Manish Godara Date: Fri, 19 Dec 2014 17:01:28 -0800 Subject: [PATCH 209/240] add clarification re parallel engine Change-Id: Iae10332dbfd941aa0d6887f2d394a8177c5132b6 --- doc/source/engines.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/source/engines.rst b/doc/source/engines.rst index 0c4b822f..59336017 100644 --- a/doc/source/engines.rst +++ b/doc/source/engines.rst @@ -247,6 +247,14 @@ object starts to take over and begins going through the stages listed below (for a more visual diagram/representation see the :ref:`engine state diagram `). +.. note:: + + The engine will respect the constraints imposed by the flow. For example, + if Engine is executing a :py:class:`.linear_flow.Flow` then it is + constrained by the dependency-graph which is linear in this case, and hence + using a Parallel Engine may not yield any benefits if one is looking for + concurrency. + Resumption ^^^^^^^^^^ From 072210a59bb703e655a99a5f2960408a5ecfa5e3 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 20 Jan 2015 22:48:57 -0800 Subject: [PATCH 210/240] The gathered runtime is for failures/not failures As long as a future was executed (whether or not it raised an exception or returned without exception) the runtime timing sum tracks this time so we should mention that is what it does/contains. Change-Id: I52438378c620c0cd4875995c5a3a116b271dc029 --- taskflow/types/futures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskflow/types/futures.py b/taskflow/types/futures.py index b589027b..1bdd1183 100644 --- a/taskflow/types/futures.py +++ b/taskflow/types/futures.py @@ -362,7 +362,7 @@ class ExecutorStatistics(object): @property def runtime(self): - """Total runtime of all submissions executed.""" + """Total runtime of all submissions executed (failed or not).""" return self._runtime @property From 342c59eb6d7c02d3a8654fe4040969f90cc90a70 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 21 Jan 2015 15:44:34 -0800 Subject: [PATCH 211/240] Fix persistence doc inheritance hierarchy The base persistence class moved out of the backends module and up into the root persistence module to better match the other pluggable backends; so we need to update inheritance hierarchy to reflect this adjustment. Change-Id: Ice98e424c574def0be2b6f8d60dba57cf9ca26f5 --- doc/source/persistence.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/persistence.rst b/doc/source/persistence.rst index 153018b6..9cb99896 100644 --- a/doc/source/persistence.rst +++ b/doc/source/persistence.rst @@ -283,7 +283,7 @@ Hierarchy ========= .. inheritance-diagram:: - taskflow.persistence.backends.base + taskflow.persistence.base taskflow.persistence.backends.impl_dir taskflow.persistence.backends.impl_memory taskflow.persistence.backends.impl_sqlalchemy From addc2864094286f34717a635bb73628def669151 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 19 Jan 2015 18:42:44 -0800 Subject: [PATCH 212/240] Use explicit WBE object arguments (instead of kwargs) Instead of passing around kwargs to root WBE classes and to contained classes prefer to use explicitnamed arguments that are passed around. This makes the code more obvious as to what the intended arguments are and makes it easier for error validation when other unknown arguments are passed (as well as for docs). Also moves the docs for the worker engine to be a sub-TOC under the main engine document so that it can be more easily explored and managed/found... Change-Id: I9413fad187c330fee494f0d4536cc27d9a90f0fb --- doc/source/engines.rst | 12 +++++++-- doc/source/index.rst | 5 ---- doc/source/workers.rst | 14 ++++------- taskflow/engines/worker_based/executor.py | 13 +++++++--- taskflow/engines/worker_based/proxy.py | 25 ++++++++----------- taskflow/engines/worker_based/server.py | 12 ++++++--- .../tests/unit/worker_based/test_executor.py | 10 +++++--- .../tests/unit/worker_based/test_proxy.py | 10 ++++---- .../tests/unit/worker_based/test_server.py | 8 ++++-- .../tests/unit/worker_based/test_worker.py | 3 ++- 10 files changed, 62 insertions(+), 50 deletions(-) diff --git a/doc/source/engines.rst b/doc/source/engines.rst index 0231ae86..e98eacde 100644 --- a/doc/source/engines.rst +++ b/doc/source/engines.rst @@ -187,8 +187,16 @@ Worker-based **Engine type**: ``'worker-based'`` -For more information, please see :doc:`workers ` for more details on -how the worker based engine operates (and the design decisions behind it). +.. note:: Since this engine is significantly more complicated (and + different) then the others we thought it appropriate to devote a + whole documentation section to it. + +For further information, please refer to the the following: + +.. toctree:: + :maxdepth: 2 + + workers How they run ============ diff --git a/doc/source/index.rst b/doc/source/index.rst index 657a08be..7ab0fedd 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -29,11 +29,6 @@ Contents jobs conductors -.. toctree:: - :hidden: - - workers - Examples -------- diff --git a/doc/source/workers.rst b/doc/source/workers.rst index 373615ef..cabe7c5b 100644 --- a/doc/source/workers.rst +++ b/doc/source/workers.rst @@ -1,7 +1,3 @@ -------- -Workers -------- - Overview ======== @@ -98,9 +94,9 @@ Use-cases Design ====== -There are two communication sides, the *executor* and *worker* that communicate -using a proxy component. The proxy is designed to accept/publish messages -from/into a named exchange. +There are two communication sides, the *executor* (and associated engine +derivative) and *worker* that communicate using a proxy component. The proxy +is designed to accept/publish messages from/into a named exchange. High level architecture ----------------------- @@ -390,7 +386,7 @@ Limitations Interfaces ========== -.. automodule:: taskflow.engines.worker_based.worker .. automodule:: taskflow.engines.worker_based.engine -.. automodule:: taskflow.engines.worker_based.proxy .. automodule:: taskflow.engines.worker_based.executor +.. automodule:: taskflow.engines.worker_based.proxy +.. automodule:: taskflow.engines.worker_based.worker diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index b07c73f8..cdc63614 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -67,14 +67,16 @@ class WorkerTaskExecutor(executor.TaskExecutor): """Executes tasks on remote workers.""" def __init__(self, uuid, exchange, topics, - transition_timeout=pr.REQUEST_TIMEOUT, **kwargs): + transition_timeout=pr.REQUEST_TIMEOUT, + url=None, transport=None, transport_options=None, + retry_options=None): self._uuid = uuid self._topics = topics self._requests_cache = cache.RequestsCache() self._transition_timeout = transition_timeout self._workers_cache = cache.WorkersCache() self._workers_arrival = threading.Condition() - handlers = { + type_handlers = { pr.NOTIFY: [ self._process_notify, functools.partial(pr.Notify.validate, response=True), @@ -84,8 +86,11 @@ class WorkerTaskExecutor(executor.TaskExecutor): pr.Response.validate, ], } - self._proxy = proxy.Proxy(uuid, exchange, handlers, - self._on_wait, **kwargs) + self._proxy = proxy.Proxy(uuid, exchange, type_handlers, + on_wait=self._on_wait, url=url, + transport=transport, + transport_options=transport_options, + retry_options=retry_options) self._proxy_thread = None self._periodic = PeriodicWorker(tt.Timeout(pr.NOTIFY_PERIOD), [self._notify_topics]) diff --git a/taskflow/engines/worker_based/proxy.py b/taskflow/engines/worker_based/proxy.py index 9a3b8e09..1770d562 100644 --- a/taskflow/engines/worker_based/proxy.py +++ b/taskflow/engines/worker_based/proxy.py @@ -64,10 +64,12 @@ class Proxy(object): # value is valid... _RETRY_INT_OPTS = frozenset(['max_retries']) - def __init__(self, topic, exchange_name, type_handlers, on_wait=None, - **kwargs): + def __init__(self, topic, exchange, type_handlers, + on_wait=None, url=None, + transport=None, transport_options=None, + retry_options=None): self._topic = topic - self._exchange_name = exchange_name + self._exchange_name = exchange self._on_wait = on_wait self._running = threading_utils.Event() self._dispatcher = dispatcher.TypeDispatcher(type_handlers) @@ -76,18 +78,13 @@ class Proxy(object): # running, otherwise requeue them. lambda data, message: not self.is_running) - # TODO(harlowja): make these keyword arguments explict... - url = kwargs.get('url') - transport = kwargs.get('transport') - transport_opts = kwargs.get('transport_options') ensure_options = self._DEFAULT_RETRY_OPTIONS.copy() - if 'retry_options' in kwargs and kwargs['retry_options'] is not None: + if retry_options is not None: # Override the defaults with any user provided values... - usr_retry_options = kwargs['retry_options'] for k in set(six.iterkeys(ensure_options)): - if k in usr_retry_options: + if k in retry_options: # Ensure that the right type is passed in... - val = usr_retry_options[k] + val = retry_options[k] if k in self._RETRY_INT_OPTS: tmp_val = int(val) else: @@ -100,14 +97,14 @@ class Proxy(object): self._ensure_options = ensure_options self._drain_events_timeout = DRAIN_EVENTS_PERIOD - if transport == 'memory' and transport_opts: - polling_interval = transport_opts.get('polling_interval') + if transport == 'memory' and transport_options: + polling_interval = transport_options.get('polling_interval') if polling_interval is not None: self._drain_events_timeout = polling_interval # create connection self._conn = kombu.Connection(url, transport=transport, - transport_options=transport_opts) + transport_options=transport_options) # create exchange self._exchange = kombu.Exchange(name=self._exchange_name, diff --git a/taskflow/engines/worker_based/server.py b/taskflow/engines/worker_based/server.py index bb7a97d2..6e64f1cd 100644 --- a/taskflow/engines/worker_based/server.py +++ b/taskflow/engines/worker_based/server.py @@ -45,8 +45,10 @@ def delayed(executor): class Server(object): """Server implementation that waits for incoming tasks requests.""" - def __init__(self, topic, exchange, executor, endpoints, **kwargs): - handlers = { + def __init__(self, topic, exchange, executor, endpoints, + url=None, transport=None, transport_options=None, + retry_options=None): + type_handlers = { pr.NOTIFY: [ delayed(executor)(self._process_notify), functools.partial(pr.Notify.validate, response=False), @@ -56,8 +58,10 @@ class Server(object): pr.Request.validate, ], } - self._proxy = proxy.Proxy(topic, exchange, handlers, - on_wait=None, **kwargs) + self._proxy = proxy.Proxy(topic, exchange, type_handlers, + url=url, transport=transport, + transport_options=transport_options, + retry_options=retry_options) self._topic = topic self._endpoints = dict([(endpoint.name, endpoint) for endpoint in endpoints]) diff --git a/taskflow/tests/unit/worker_based/test_executor.py b/taskflow/tests/unit/worker_based/test_executor.py index 4e0c38bc..cdb421d1 100644 --- a/taskflow/tests/unit/worker_based/test_executor.py +++ b/taskflow/tests/unit/worker_based/test_executor.py @@ -84,10 +84,13 @@ class TestWorkerTaskExecutor(test.MockTestCase): def test_creation(self): ex = self.executor(reset_master_mock=False) - master_mock_calls = [ mock.call.Proxy(self.executor_uuid, self.executor_exchange, - mock.ANY, ex._on_wait, url=self.broker_url) + mock.ANY, on_wait=ex._on_wait, + url=self.broker_url, transport=mock.ANY, + transport_options=mock.ANY, + retry_options=mock.ANY + ) ] self.assertEqual(self.master_mock.mock_calls, master_mock_calls) @@ -250,8 +253,7 @@ class TestWorkerTaskExecutor(test.MockTestCase): self.assertEqual(expected_calls, self.master_mock.mock_calls) def test_execute_task_topic_not_found(self): - workers_info = {self.executor_topic: ['']} - ex = self.executor(workers_info=workers_info) + ex = self.executor() ex.execute_task(self.task, self.task_uuid, self.task_args) expected_calls = [ diff --git a/taskflow/tests/unit/worker_based/test_proxy.py b/taskflow/tests/unit/worker_based/test_proxy.py index daf9b60e..7ec91780 100644 --- a/taskflow/tests/unit/worker_based/test_proxy.py +++ b/taskflow/tests/unit/worker_based/test_proxy.py @@ -28,7 +28,7 @@ class TestProxy(test.MockTestCase): super(TestProxy, self).setUp() self.topic = 'test-topic' self.broker_url = 'test-url' - self.exchange_name = 'test-exchange' + self.exchange = 'test-exchange' self.timeout = 5 self.de_period = proxy.DRAIN_EVENTS_PERIOD @@ -72,7 +72,7 @@ class TestProxy(test.MockTestCase): self.resetMasterMock() def _queue_name(self, topic): - return "%s_%s" % (self.exchange_name, topic) + return "%s_%s" % (self.exchange, topic) def proxy_start_calls(self, calls, exc_type=mock.ANY): return [ @@ -119,7 +119,7 @@ class TestProxy(test.MockTestCase): def proxy(self, reset_master_mock=False, **kwargs): proxy_kwargs = dict(topic=self.topic, - exchange_name=self.exchange_name, + exchange=self.exchange, url=self.broker_url, type_handlers={}) proxy_kwargs.update(kwargs) @@ -134,7 +134,7 @@ class TestProxy(test.MockTestCase): master_mock_calls = [ mock.call.Connection(self.broker_url, transport=None, transport_options=None), - mock.call.Exchange(name=self.exchange_name, + mock.call.Exchange(name=self.exchange, durable=False, auto_delete=True) ] @@ -147,7 +147,7 @@ class TestProxy(test.MockTestCase): master_mock_calls = [ mock.call.Connection(self.broker_url, transport='memory', transport_options=transport_opts), - mock.call.Exchange(name=self.exchange_name, + mock.call.Exchange(name=self.exchange, durable=False, auto_delete=True) ] diff --git a/taskflow/tests/unit/worker_based/test_server.py b/taskflow/tests/unit/worker_based/test_server.py index 7da5b432..7a77e8df 100644 --- a/taskflow/tests/unit/worker_based/test_server.py +++ b/taskflow/tests/unit/worker_based/test_server.py @@ -86,7 +86,9 @@ class TestServer(test.MockTestCase): # check calls master_mock_calls = [ mock.call.Proxy(self.server_topic, self.server_exchange, - mock.ANY, url=self.broker_url, on_wait=mock.ANY) + mock.ANY, url=self.broker_url, + transport=mock.ANY, transport_options=mock.ANY, + retry_options=mock.ANY) ] self.master_mock.assert_has_calls(master_mock_calls) self.assertEqual(len(s._endpoints), 3) @@ -97,7 +99,9 @@ class TestServer(test.MockTestCase): # check calls master_mock_calls = [ mock.call.Proxy(self.server_topic, self.server_exchange, - mock.ANY, url=self.broker_url, on_wait=mock.ANY) + mock.ANY, url=self.broker_url, + transport=mock.ANY, transport_options=mock.ANY, + retry_options=mock.ANY) ] self.master_mock.assert_has_calls(master_mock_calls) self.assertEqual(len(s._endpoints), len(self.endpoints)) diff --git a/taskflow/tests/unit/worker_based/test_worker.py b/taskflow/tests/unit/worker_based/test_worker.py index c8ab9185..a572beb4 100644 --- a/taskflow/tests/unit/worker_based/test_worker.py +++ b/taskflow/tests/unit/worker_based/test_worker.py @@ -82,7 +82,8 @@ class TestWorker(test.MockTestCase): master_mock_calls = [ mock.call.executor_class(10), mock.call.Server(self.topic, self.exchange, - self.executor_inst_mock, [], url=self.broker_url) + self.executor_inst_mock, [], + url=self.broker_url) ] self.assertEqual(self.master_mock.mock_calls, master_mock_calls) From 5773fb09e68030acbd318e45f795de4a6a37caee Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 21 Jan 2015 17:07:13 -0800 Subject: [PATCH 213/240] Use a class provided logger before falling back to module If a subclass overrides the logging listeners and provides a class specific logger (under the attribute named _LOGGER) try to use that before using the taskflow specific logging module (and only use this one as a last resort). This allows subclasses to easily override the default logger without having to continually pass in a constructor argument to do the same. Change-Id: I91fea3db6cdd1dfb39963ff9589fd530fe087278 --- taskflow/listeners/logging.py | 33 ++++++++++++++++++--------------- taskflow/utils/misc.py | 8 ++++++++ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/taskflow/listeners/logging.py b/taskflow/listeners/logging.py index 3e139587..b2d4d344 100644 --- a/taskflow/listeners/logging.py +++ b/taskflow/listeners/logging.py @@ -24,6 +24,7 @@ from taskflow.listeners import base from taskflow import logging from taskflow import states from taskflow.types import failure +from taskflow.utils import misc LOG = logging.getLogger(__name__) @@ -48,10 +49,14 @@ class LoggingListener(base.DumpingListener): It listens for task and flow notifications and writes those notifications to a provided logger, or logger of its module - (``taskflow.listeners.logging``) if none is provided. The log level - can also be configured, ``logging.DEBUG`` is used by default when none - is provided. + (``taskflow.listeners.logging``) if none is provided (and no class + attribute is overriden). The log level can also be + configured, ``logging.DEBUG`` is used by default when none is provided. """ + + #: Default logger to use if one is not provided on construction. + _LOGGER = None + def __init__(self, engine, task_listen_for=base.DEFAULT_LISTEN_FOR, flow_listen_for=base.DEFAULT_LISTEN_FOR, @@ -61,10 +66,7 @@ class LoggingListener(base.DumpingListener): super(LoggingListener, self).__init__( engine, task_listen_for=task_listen_for, flow_listen_for=flow_listen_for, retry_listen_for=retry_listen_for) - if not log: - self._logger = LOG - else: - self._logger = log + self._logger = misc.pick_first_not_none(log, self._LOGGER, LOG) self._level = level def _dump(self, message, *args, **kwargs): @@ -76,10 +78,11 @@ class DynamicLoggingListener(base.Listener): It listens for task and flow notifications and writes those notifications to a provided logger, or logger of its module - (``taskflow.listeners.logging``) if none is provided. The log level - can *slightly* be configured and ``logging.DEBUG`` or ``logging.WARNING`` - (unless overriden via a constructor parameter) will be selected - automatically based on the execution state and results produced. + (``taskflow.listeners.logging``) if none is provided (and no class + attribute is overriden). The log level can *slightly* be configured + and ``logging.DEBUG`` or ``logging.WARNING`` (unless overriden via a + constructor parameter) will be selected automatically based on the + execution state and results produced. The following flow states cause ``logging.WARNING`` (or provided level) to be used: @@ -101,6 +104,9 @@ class DynamicLoggingListener(base.Listener): provide a meaningful traceback). """ + #: Default logger to use if one is not provided on construction. + _LOGGER = None + def __init__(self, engine, task_listen_for=base.DEFAULT_LISTEN_FOR, flow_listen_for=base.DEFAULT_LISTEN_FOR, @@ -121,10 +127,7 @@ class DynamicLoggingListener(base.Listener): states.FAILURE: self._failure_level, states.REVERTED: self._failure_level, } - if not log: - self._logger = LOG - else: - self._logger = log + self._logger = misc.pick_first_not_none(log, self._LOGGER, LOG) @staticmethod def _format_failure(fail): diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 04150003..74a79867 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -112,6 +112,14 @@ def find_subclasses(locations, base_cls, exclude_hidden=True): return derived +def pick_first_not_none(*values): + """Returns first of values that is *not* None (or None if all are/were).""" + for val in values: + if val is not None: + return val + return None + + def parse_uri(uri): """Parses a uri into its components.""" # Do some basic validation before continuing... From 93d73b85c6972732892ce14fabedde4c7d4e4193 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 22 Jan 2015 14:46:47 -0800 Subject: [PATCH 214/240] Show the failure discarded (and the future intention) When DEBUG is enabled it is quite useful to show information about why a failure of a atom is being discarded (typically it is being retried) so when this happens and DEBUG is enabled add appropriate logging messages that show this decision and future intentions. Change-Id: I9aeb928ce85d057f872392b593b3e97f694474d1 --- taskflow/engines/action_engine/runner.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/taskflow/engines/action_engine/runner.py b/taskflow/engines/action_engine/runner.py index 45ee0ba6..a5bebd61 100644 --- a/taskflow/engines/action_engine/runner.py +++ b/taskflow/engines/action_engine/runner.py @@ -143,8 +143,24 @@ class _MachineBuilder(object): try: event, result = fut.result() retain = self._completer.complete(node, event, result) - if retain and isinstance(result, failure.Failure): - memory.failures.append(result) + if isinstance(result, failure.Failure): + if retain: + memory.failures.append(result) + else: + # NOTE(harlowja): avoid making any + # intention request to storage unless we are + # sure we are in DEBUG enabled logging (otherwise + # we will call this all the time even when DEBUG + # is not enabled, which would suck...) + if LOG.isEnabledFor(logging.DEBUG): + intention = self._storage.get_atom_intention( + node.name) + LOG.debug("Discarding failure '%s' (in" + " response to event '%s') under" + " completion units request during" + " completion of node '%s' (intention" + " is to %s)", result, event, + node, intention) except Exception: memory.failures.append(failure.Failure()) else: From 14695529c6dcfab1176380eaab76e9a400282593 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 22 Jan 2015 17:52:19 -0800 Subject: [PATCH 215/240] Fix leftover/remaining 'oslo.utils' usage It appears another usage of 'oslo.utils' must have merged so fix this to use the non-deprecated 'oslo_utils' import instead. Change-Id: I4e7bd64097e5e98586203beff79cb581b864b20c --- taskflow/types/futures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskflow/types/futures.py b/taskflow/types/futures.py index 1bdd1183..128f9aa5 100644 --- a/taskflow/types/futures.py +++ b/taskflow/types/futures.py @@ -20,7 +20,7 @@ import threading from concurrent import futures as _futures from concurrent.futures import process as _process from concurrent.futures import thread as _thread -from oslo.utils import reflection +from oslo_utils import reflection try: from eventlet.green import threading as greenthreading From 45ef595fdecb2b755d89fa454c7af5b69f1b48aa Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 26 Jun 2014 16:15:05 -0700 Subject: [PATCH 216/240] Tidy up the WBE cache (now WBE types) module Instead of using the expiring cache type as a means to store worker information just avoid using that type since we don't support expiry in the first place on worker information and use a worker container and a worker object that we can later extend as needed. Also add on clear methods to the cache type that will be used when the WBE executor stop occurs. This ensures we clear out the worker information and any unfinished requests. Change-Id: I6a520376eff1e8a6edcef0a59f2d8b9c0eb15752 --- taskflow/engines/worker_based/cache.py | 48 ---- taskflow/engines/worker_based/executor.py | 90 ++------ taskflow/engines/worker_based/protocol.py | 15 +- taskflow/engines/worker_based/types.py | 217 ++++++++++++++++++ .../tests/unit/worker_based/test_protocol.py | 2 +- .../tests/unit/worker_based/test_types.py | 139 +++++++++++ taskflow/types/cache.py | 15 ++ 7 files changed, 404 insertions(+), 122 deletions(-) delete mode 100644 taskflow/engines/worker_based/cache.py create mode 100644 taskflow/engines/worker_based/types.py create mode 100644 taskflow/tests/unit/worker_based/test_types.py diff --git a/taskflow/engines/worker_based/cache.py b/taskflow/engines/worker_based/cache.py deleted file mode 100644 index 3b00890d..00000000 --- a/taskflow/engines/worker_based/cache.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import random - -import six - -from taskflow.engines.worker_based import protocol as pr -from taskflow.types import cache as base - - -class RequestsCache(base.ExpiringCache): - """Represents a thread-safe requests cache.""" - - def get_waiting_requests(self, tasks): - """Get list of waiting requests by tasks.""" - waiting_requests = [] - with self._lock: - for request in six.itervalues(self._data): - if request.state == pr.WAITING and request.task_cls in tasks: - waiting_requests.append(request) - return waiting_requests - - -class WorkersCache(base.ExpiringCache): - """Represents a thread-safe workers cache.""" - - def get_topic_by_task(self, task): - """Get topic for a given task.""" - available_topics = [] - with self._lock: - for topic, tasks in six.iteritems(self._data): - if task in tasks: - available_topics.append(topic) - return random.choice(available_topics) if available_topics else None diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index cdc63614..cda37458 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -15,15 +15,13 @@ # under the License. import functools -import threading -from oslo_utils import reflection from oslo_utils import timeutils from taskflow.engines.action_engine import executor -from taskflow.engines.worker_based import cache from taskflow.engines.worker_based import protocol as pr from taskflow.engines.worker_based import proxy +from taskflow.engines.worker_based import types as wt from taskflow import exceptions as exc from taskflow import logging from taskflow import task as task_atom @@ -34,35 +32,6 @@ from taskflow.utils import threading_utils as tu LOG = logging.getLogger(__name__) -class PeriodicWorker(object): - """Calls a set of functions when activated periodically. - - NOTE(harlowja): the provided timeout object determines the periodicity. - """ - def __init__(self, timeout, functors): - self._timeout = timeout - self._functors = [] - for f in functors: - self._functors.append((f, reflection.get_callable_name(f))) - - def start(self): - while not self._timeout.is_stopped(): - for (f, f_name) in self._functors: - LOG.debug("Calling periodic function '%s'", f_name) - try: - f() - except Exception: - LOG.warn("Failed to call periodic function '%s'", f_name, - exc_info=True) - self._timeout.wait() - - def stop(self): - self._timeout.interrupt() - - def reset(self): - self._timeout.reset() - - class WorkerTaskExecutor(executor.TaskExecutor): """Executes tasks on remote workers.""" @@ -72,10 +41,9 @@ class WorkerTaskExecutor(executor.TaskExecutor): retry_options=None): self._uuid = uuid self._topics = topics - self._requests_cache = cache.RequestsCache() + self._requests_cache = wt.RequestsCache() + self._workers = wt.TopicWorkers() self._transition_timeout = transition_timeout - self._workers_cache = cache.WorkersCache() - self._workers_arrival = threading.Condition() type_handlers = { pr.NOTIFY: [ self._process_notify, @@ -92,8 +60,8 @@ class WorkerTaskExecutor(executor.TaskExecutor): transport_options=transport_options, retry_options=retry_options) self._proxy_thread = None - self._periodic = PeriodicWorker(tt.Timeout(pr.NOTIFY_PERIOD), - [self._notify_topics]) + self._periodic = wt.PeriodicWorker(tt.Timeout(pr.NOTIFY_PERIOD), + [self._notify_topics]) self._periodic_thread = None def _process_notify(self, notify, message): @@ -104,16 +72,15 @@ class WorkerTaskExecutor(executor.TaskExecutor): tasks = notify['tasks'] # Add worker info to the cache - LOG.debug("Received that tasks %s can be processed by topic '%s'", - tasks, topic) - with self._workers_arrival: - self._workers_cache[topic] = tasks - self._workers_arrival.notify_all() + worker = self._workers.add(topic, tasks) + LOG.debug("Received notification about worker '%s' (%s" + " total workers are currently known)", worker, + len(self._workers)) # Publish waiting requests - for request in self._requests_cache.get_waiting_requests(tasks): + for request in self._requests_cache.get_waiting_requests(worker): if request.transition_and_log_error(pr.PENDING, logger=LOG): - self._publish_request(request, topic) + self._publish_request(request, worker) def _process_response(self, response, message): """Process response from remote side.""" @@ -147,7 +114,7 @@ class WorkerTaskExecutor(executor.TaskExecutor): del self._requests_cache[request.uuid] request.set_result(**response.data) else: - LOG.warning("Unexpected response status: '%s'", + LOG.warning("Unexpected response status '%s'", response.state) else: LOG.debug("Request with id='%s' not found", task_uuid) @@ -196,16 +163,16 @@ class WorkerTaskExecutor(executor.TaskExecutor): progress_callback) request.result.add_done_callback(lambda fut: cleaner()) - # Get task's topic and publish request if topic was found. - topic = self._workers_cache.get_topic_by_task(request.task_cls) - if topic is not None: + # Get task's worker and publish request if worker was found. + worker = self._workers.get_worker_for_task(task) + if worker is not None: # NOTE(skudriashev): Make sure request is set to the PENDING state # before putting it into the requests cache to prevent the notify # processing thread get list of waiting requests and publish it # before it is published here, so it wouldn't be published twice. if request.transition_and_log_error(pr.PENDING, logger=LOG): self._requests_cache[request.uuid] = request - self._publish_request(request, topic) + self._publish_request(request, worker) else: LOG.debug("Delaying submission of '%s', no currently known" " worker/s available to process it", request) @@ -213,14 +180,14 @@ class WorkerTaskExecutor(executor.TaskExecutor): return request.result - def _publish_request(self, request, topic): + def _publish_request(self, request, worker): """Publish request to a given topic.""" - LOG.debug("Submitting execution of '%s' to topic '%s' (expecting" + LOG.debug("Submitting execution of '%s' to worker '%s' (expecting" " response identified by reply_to=%s and" - " correlation_id=%s)", request, topic, self._uuid, + " correlation_id=%s)", request, worker, self._uuid, request.uuid) try: - self._proxy.publish(request, topic, + self._proxy.publish(request, worker.topic, reply_to=self._uuid, correlation_id=request.uuid) except Exception: @@ -255,20 +222,7 @@ class WorkerTaskExecutor(executor.TaskExecutor): return how many workers are still needed, otherwise it will return zero. """ - if workers <= 0: - raise ValueError("Worker amount must be greater than zero") - w = None - if timeout is not None: - w = tt.StopWatch(timeout).start() - with self._workers_arrival: - while len(self._workers_cache) < workers: - if w is not None and w.expired(): - return workers - len(self._workers_cache) - timeout = None - if w is not None: - timeout = w.leftover() - self._workers_arrival.wait(timeout) - return 0 + return self._workers.wait_for_workers(workers=workers, timeout=timeout) def start(self): """Starts proxy thread and associated topic notification thread.""" @@ -291,3 +245,5 @@ class WorkerTaskExecutor(executor.TaskExecutor): self._proxy.stop() self._proxy_thread.join() self._proxy_thread = None + self._requests_cache.clear(self._handle_expired_request) + self._workers.clear() diff --git a/taskflow/engines/worker_based/protocol.py b/taskflow/engines/worker_based/protocol.py index 19813d40..0bb57a04 100644 --- a/taskflow/engines/worker_based/protocol.py +++ b/taskflow/engines/worker_based/protocol.py @@ -221,7 +221,6 @@ class Request(Message): def __init__(self, task, uuid, action, arguments, timeout, **kwargs): self._task = task - self._task_cls = reflection.get_class_name(task) self._uuid = uuid self._action = action self._event = ACTION_TO_EVENT[action] @@ -248,8 +247,8 @@ class Request(Message): return self._uuid @property - def task_cls(self): - return self._task_cls + def task(self): + return self._task @property def state(self): @@ -281,9 +280,13 @@ class Request(Message): convert all `failure.Failure` objects into dictionaries (which will then be reconstituted by the receiver). """ - request = dict(task_cls=self._task_cls, task_name=self._task.name, - task_version=self._task.version, action=self._action, - arguments=self._arguments) + request = { + 'task_cls': reflection.get_class_name(self._task), + 'task_name': self._task.name, + 'task_version': self._task.version, + 'action': self._action, + 'arguments': self._arguments, + } if 'result' in self._kwargs: result = self._kwargs['result'] if isinstance(result, ft.Failure): diff --git a/taskflow/engines/worker_based/types.py b/taskflow/engines/worker_based/types.py new file mode 100644 index 00000000..3d8aa632 --- /dev/null +++ b/taskflow/engines/worker_based/types.py @@ -0,0 +1,217 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import itertools +import logging +import random +import threading + +from oslo.utils import reflection +import six + +from taskflow.engines.worker_based import protocol as pr +from taskflow.types import cache as base +from taskflow.types import timing as tt + +LOG = logging.getLogger(__name__) + + +class RequestsCache(base.ExpiringCache): + """Represents a thread-safe requests cache.""" + + def get_waiting_requests(self, worker): + """Get list of waiting requests that the given worker can satisfy.""" + waiting_requests = [] + with self._lock: + for request in six.itervalues(self._data): + if request.state == pr.WAITING \ + and worker.performs(request.task): + waiting_requests.append(request) + return waiting_requests + + +# TODO(harlowja): this needs to be made better, once +# https://blueprints.launchpad.net/taskflow/+spec/wbe-worker-info is finally +# implemented we can go about using that instead. +class TopicWorker(object): + """A (read-only) worker and its relevant information + useful methods.""" + + _NO_IDENTITY = object() + + def __init__(self, topic, tasks, identity=_NO_IDENTITY): + self.tasks = [] + for task in tasks: + if not isinstance(task, six.string_types): + task = reflection.get_class_name(task) + self.tasks.append(task) + self.topic = topic + self.identity = identity + + def performs(self, task): + if not isinstance(task, six.string_types): + task = reflection.get_class_name(task) + return task in self.tasks + + def __eq__(self, other): + if not isinstance(other, TopicWorker): + return NotImplemented + if len(other.tasks) != len(self.tasks): + return False + if other.topic != self.topic: + return False + for task in other.tasks: + if not self.performs(task): + return False + # If one of the identity equals _NO_IDENTITY, then allow it to match... + if self._NO_IDENTITY in (self.identity, other.identity): + return True + else: + return other.identity == self.identity + + def __repr__(self): + r = reflection.get_class_name(self, fully_qualified=False) + if self.identity is not self._NO_IDENTITY: + r += "(identity=%s, tasks=%s, topic=%s)" % (self.identity, + self.tasks, self.topic) + else: + r += "(identity=*, tasks=%s, topic=%s)" % (self.tasks, self.topic) + return r + + +class TopicWorkers(object): + """A collection of topic based workers.""" + + @staticmethod + def _match_worker(task, available_workers): + """Select a worker (from geq 1 workers) that can best perform the task. + + NOTE(harlowja): this method will be activated when there exists + one one greater than one potential workers that can perform a task, + the arguments provided will be the potential workers located and the + task that is being requested to perform and the result should be one + of those workers using whatever best-fit algorithm is possible (or + random at the least). + """ + if len(available_workers) == 1: + return available_workers[0] + else: + return random.choice(available_workers) + + def __init__(self): + self._workers = {} + self._cond = threading.Condition() + # Used to name workers with more useful identities... + self._counter = itertools.count() + + def __len__(self): + return len(self._workers) + + def _next_worker(self, topic, tasks, temporary=False): + if not temporary: + return TopicWorker(topic, tasks, + identity=six.next(self._counter)) + else: + return TopicWorker(topic, tasks) + + def add(self, topic, tasks): + """Adds/updates a worker for the topic for the given tasks.""" + with self._cond: + try: + worker = self._workers[topic] + # Check if we already have an equivalent worker, if so just + # return it... + if worker == self._next_worker(topic, tasks, temporary=True): + return worker + # This *fall through* is done so that if someone is using an + # active worker object that already exists that we just create + # a new one; so that the existing object doesn't get + # affected (workers objects are supposed to be immutable). + except KeyError: + pass + worker = self._next_worker(topic, tasks) + self._workers[topic] = worker + self._cond.notify_all() + return worker + + def wait_for_workers(self, workers=1, timeout=None): + """Waits for geq workers to notify they are ready to do work. + + NOTE(harlowja): if a timeout is provided this function will wait + until that timeout expires, if the amount of workers does not reach + the desired amount of workers before the timeout expires then this will + return how many workers are still needed, otherwise it will + return zero. + """ + if workers <= 0: + raise ValueError("Worker amount must be greater than zero") + w = None + if timeout is not None: + w = tt.StopWatch(timeout).start() + with self._cond: + while len(self._workers) < workers: + if w is not None and w.expired(): + return max(0, workers - len(self._workers)) + timeout = None + if w is not None: + timeout = w.leftover() + self._cond.wait(timeout) + return 0 + + def get_worker_for_task(self, task): + """Gets a worker that can perform a given task.""" + available_workers = [] + with self._cond: + for worker in six.itervalues(self._workers): + if worker.performs(task): + available_workers.append(worker) + if available_workers: + return self._match_worker(task, available_workers) + else: + return None + + def clear(self): + with self._cond: + self._workers.clear() + self._cond.notify_all() + + +class PeriodicWorker(object): + """Calls a set of functions when activated periodically. + + NOTE(harlowja): the provided timeout object determines the periodicity. + """ + def __init__(self, timeout, functors): + self._timeout = timeout + self._functors = [] + for f in functors: + self._functors.append((f, reflection.get_callable_name(f))) + + def start(self): + while not self._timeout.is_stopped(): + for (f, f_name) in self._functors: + LOG.debug("Calling periodic function '%s'", f_name) + try: + f() + except Exception: + LOG.warn("Failed to call periodic function '%s'", f_name, + exc_info=True) + self._timeout.wait() + + def stop(self): + self._timeout.interrupt() + + def reset(self): + self._timeout.reset() diff --git a/taskflow/tests/unit/worker_based/test_protocol.py b/taskflow/tests/unit/worker_based/test_protocol.py index 5356fb99..e5da38f2 100644 --- a/taskflow/tests/unit/worker_based/test_protocol.py +++ b/taskflow/tests/unit/worker_based/test_protocol.py @@ -136,7 +136,7 @@ class TestProtocol(test.TestCase): def test_creation(self): request = self.request() self.assertEqual(request.uuid, self.task_uuid) - self.assertEqual(request.task_cls, self.task.name) + self.assertEqual(request.task, self.task) self.assertIsInstance(request.result, futures.Future) self.assertFalse(request.result.done()) diff --git a/taskflow/tests/unit/worker_based/test_types.py b/taskflow/tests/unit/worker_based/test_types.py new file mode 100644 index 00000000..6541575a --- /dev/null +++ b/taskflow/tests/unit/worker_based/test_types.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime +import threading +import time + +from oslo.utils import reflection +from oslo.utils import timeutils + +from taskflow.engines.worker_based import protocol as pr +from taskflow.engines.worker_based import types as worker_types +from taskflow import test +from taskflow.tests import utils +from taskflow.types import latch +from taskflow.types import timing + + +class TestWorkerTypes(test.TestCase): + + def setUp(self): + super(TestWorkerTypes, self).setUp() + self.task = utils.DummyTask() + self.task_uuid = 'task-uuid' + self.task_action = 'execute' + self.task_args = {'a': 'a'} + self.timeout = 60 + + def request(self, **kwargs): + request_kwargs = dict(task=self.task, + uuid=self.task_uuid, + action=self.task_action, + arguments=self.task_args, + progress_callback=None, + timeout=self.timeout) + request_kwargs.update(kwargs) + return pr.Request(**request_kwargs) + + def test_requests_cache_expiry(self): + # Mock out the calls the underlying objects will soon use to return + # times that we can control more easily... + now = timeutils.utcnow() + overrides = [ + now, + now, + now + datetime.timedelta(seconds=1), + now + datetime.timedelta(seconds=self.timeout + 1), + ] + timeutils.set_time_override(overrides) + self.addCleanup(timeutils.clear_time_override) + + cache = worker_types.RequestsCache() + cache[self.task_uuid] = self.request() + cache.cleanup() + self.assertEqual(1, len(cache)) + cache.cleanup() + self.assertEqual(0, len(cache)) + + def test_requests_cache_match(self): + cache = worker_types.RequestsCache() + cache[self.task_uuid] = self.request() + cache['task-uuid-2'] = self.request(task=utils.NastyTask(), + uuid='task-uuid-2') + worker = worker_types.TopicWorker("dummy-topic", [utils.DummyTask], + identity="dummy") + matches = cache.get_waiting_requests(worker) + self.assertEqual(1, len(matches)) + self.assertEqual(2, len(cache)) + + def test_topic_worker(self): + worker = worker_types.TopicWorker("dummy-topic", + [utils.DummyTask], identity="dummy") + self.assertTrue(worker.performs(utils.DummyTask)) + self.assertFalse(worker.performs(utils.NastyTask)) + self.assertEqual('dummy', worker.identity) + self.assertEqual('dummy-topic', worker.topic) + + def test_single_topic_workers(self): + workers = worker_types.TopicWorkers() + w = workers.add('dummy-topic', [utils.DummyTask]) + self.assertIsNotNone(w) + self.assertEqual(1, len(workers)) + w2 = workers.get_worker_for_task(utils.DummyTask) + self.assertEqual(w.identity, w2.identity) + + def test_multi_same_topic_workers(self): + workers = worker_types.TopicWorkers() + w = workers.add('dummy-topic', [utils.DummyTask]) + self.assertIsNotNone(w) + w2 = workers.add('dummy-topic-2', [utils.DummyTask]) + self.assertIsNotNone(w2) + w3 = workers.get_worker_for_task( + reflection.get_class_name(utils.DummyTask)) + self.assertIn(w3.identity, [w.identity, w2.identity]) + + def test_multi_different_topic_workers(self): + workers = worker_types.TopicWorkers() + added = [] + added.append(workers.add('dummy-topic', [utils.DummyTask])) + added.append(workers.add('dummy-topic-2', [utils.DummyTask])) + added.append(workers.add('dummy-topic-3', [utils.NastyTask])) + self.assertEqual(3, len(workers)) + w = workers.get_worker_for_task(utils.NastyTask) + self.assertEqual(added[-1].identity, w.identity) + w = workers.get_worker_for_task(utils.DummyTask) + self.assertIn(w.identity, [w_a.identity for w_a in added[0:2]]) + + def test_periodic_worker(self): + barrier = latch.Latch(5) + to = timing.Timeout(0.01) + called_at = [] + + def callee(): + barrier.countdown() + if barrier.needed == 0: + to.interrupt() + called_at.append(time.time()) + + w = worker_types.PeriodicWorker(to, [callee]) + t = threading.Thread(target=w.start) + t.start() + t.join() + + self.assertEqual(0, barrier.needed) + self.assertEqual(5, len(called_at)) + self.assertTrue(to.is_stopped()) diff --git a/taskflow/types/cache.py b/taskflow/types/cache.py index d9b14910..802bc610 100644 --- a/taskflow/types/cache.py +++ b/taskflow/types/cache.py @@ -54,6 +54,21 @@ class ExpiringCache(object): with self._lock: del self._data[key] + def clear(self, on_cleared_callback=None): + """Removes all keys & values from the cache.""" + cleared_items = [] + with self._lock: + if on_cleared_callback is not None: + cleared_items.extend(six.iteritems(self._data)) + self._data.clear() + if on_cleared_callback is not None: + arg_c = len(reflection.get_callable_args(on_cleared_callback)) + for (k, v) in cleared_items: + if arg_c == 2: + on_cleared_callback(k, v) + else: + on_cleared_callback(v) + def cleanup(self, on_expired_callback=None): """Delete out-dated keys & values from the cache.""" with self._lock: From 55ad11f278e2eb4133486fa817eca292ec1fec8a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 21 Jan 2015 13:08:33 -0800 Subject: [PATCH 217/240] Add a WBE request state diagram + explanation To make it more clear what the WBE request states are and what they imply/mean add the appropriate documentation and diagram that explains it and its states/concepts. Change-Id: If25b5c6402aff6e294886cc6c5f248413183c4e4 --- doc/source/img/wbe_request_states.svg | 8 +++++ doc/source/workers.rst | 48 +++++++++++++++++++++++++-- tools/generate_states.sh | 4 +++ tools/state_graph.py | 19 +++++++++-- 4 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 doc/source/img/wbe_request_states.svg diff --git a/doc/source/img/wbe_request_states.svg b/doc/source/img/wbe_request_states.svg new file mode 100644 index 00000000..da2c9d30 --- /dev/null +++ b/doc/source/img/wbe_request_states.svg @@ -0,0 +1,8 @@ + + + + + +WBE requests statesWAITINGPENDINGFAILURERUNNINGSUCCESSstart + diff --git a/doc/source/workers.rst b/doc/source/workers.rst index cabe7c5b..f02caca7 100644 --- a/doc/source/workers.rst +++ b/doc/source/workers.rst @@ -13,7 +13,6 @@ connected via `amqp`_ (or other supported `kombu`_ transports). production ready. .. _blueprint page: https://blueprints.launchpad.net/taskflow?searchtext=wbe -.. _kombu: http://kombu.readthedocs.org/ Terminology ----------- @@ -285,10 +284,53 @@ When **reverting:** ] } +Request state transitions +------------------------- + +.. image:: img/wbe_request_states.svg + :width: 660px + :align: left + :alt: WBE request state transitions + +**WAITING** - Request placed on queue (or other `kombu`_ message bus/transport) +but not *yet* consumed. + +**PENDING** - Worker accepted request and is pending to run using its +executor (threads, processes, or other). + +**FAILURE** - Worker failed after running request (due to task exeception) or +no worker moved/started executing (by placing the request into ``RUNNING`` +state) with-in specified time span (this defaults to 60 seconds unless +overriden). + +**RUNNING** - Workers executor (using threads, processes...) has started to +run requested task (once this state is transitioned to any request timeout no +longer becomes applicable; since at this point it is unknown how long a task +will run since it can not be determined if a task is just taking a long time +or has failed). + +**SUCCESS** - Worker finished running task without exception. + +.. note:: + + During the ``WAITING`` and ``PENDING`` stages the engine keeps track + of how long the request has been *alive* for and if a timeout is reached + the request will automatically transition to ``FAILURE`` and any further + transitions from a worker will be disallowed (for example, if a worker + accepts the request in the future and sets the task to ``PENDING`` this + transition will be logged and ignored). This timeout can be adjusted and/or + removed by setting the engine ``transition_timeout`` option to a + higher/lower value or by setting it to ``None`` (to remove the timeout + completely). In the future this will be improved to be more dynamic + by implementing the blueprints associated with `failover`_ and + `info/resilence`_. + +.. _failover: https://blueprints.launchpad.net/taskflow/+spec/wbe-worker-failover +.. _info/resilence: https://blueprints.launchpad.net/taskflow/+spec/wbe-worker-info + Usage ===== - Workers ------- @@ -390,3 +432,5 @@ Interfaces .. automodule:: taskflow.engines.worker_based.executor .. automodule:: taskflow.engines.worker_based.proxy .. automodule:: taskflow.engines.worker_based.worker + +.. _kombu: http://kombu.readthedocs.org/ diff --git a/tools/generate_states.sh b/tools/generate_states.sh index 2da75817..308c6400 100755 --- a/tools/generate_states.sh +++ b/tools/generate_states.sh @@ -30,3 +30,7 @@ $xsltproc $PWD/.diagram-tools/notugly.xsl /tmp/states.svg > $img_dir/engine_stat echo "---- Updating retry state diagram ----" python $script_dir/state_graph.py -r -f /tmp/states.svg $xsltproc $PWD/.diagram-tools/notugly.xsl /tmp/states.svg > $img_dir/retry_states.svg + +echo "---- Updating wbe request state diagram ----" +python $script_dir/state_graph.py -w -f /tmp/states.svg +$xsltproc $PWD/.diagram-tools/notugly.xsl /tmp/states.svg > $img_dir/wbe_request_states.svg diff --git a/tools/state_graph.py b/tools/state_graph.py index ce5a1d79..5ba9da7f 100755 --- a/tools/state_graph.py +++ b/tools/state_graph.py @@ -27,6 +27,7 @@ sys.path.insert(0, top_dir) import pydot from taskflow.engines.action_engine import runner +from taskflow.engines.worker_based import protocol from taskflow import states from taskflow.types import fsm @@ -91,6 +92,10 @@ def main(): action='store_true', help="use engine state transitions", default=False) + parser.add_option("-w", "--wbe-requests", dest="wbe_requests", + action='store_true', + help="use wbe request transitions", + default=False) parser.add_option("-T", "--format", dest="format", help="output in given format", default='svg') @@ -99,9 +104,15 @@ def main(): if options.filename is None: options.filename = 'states.%s' % options.format - types = [options.engines, options.retries, options.tasks] + types = [ + options.engines, + options.retries, + options.tasks, + options.wbe_requests, + ] if sum([int(i) for i in types]) > 1: - parser.error("Only one of task/retry/engines may be specified.") + parser.error("Only one of task/retry/engines/wbe requests" + " may be specified.") internal_states = list() ordering = 'in' @@ -120,6 +131,10 @@ def main(): source, memory = r.builder.build() internal_states.extend(runner._META_STATES) ordering = 'out' + elif options.wbe_requests: + source_type = "WBE requests" + source = make_machine(protocol.WAITING, + list(protocol._ALLOWED_TRANSITIONS), []) else: source_type = "Flow" source = make_machine(states.PENDING, From 067246771f4e7133c12ab15b0ee6aadd040f6793 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 21 Jan 2015 15:28:37 -0800 Subject: [PATCH 218/240] WBE documentation tweaks/adjustments Include the protocol code/classes (and the json schema that is used for validation) in the docs; adjust the examples to be after the protocol definition/classes. Some other small wording tweaks as well. Change-Id: Ic2987c6f4393a5398065de883bdb15ffec923451 --- doc/source/workers.rst | 114 ++++++++++++---------- taskflow/engines/worker_based/protocol.py | 29 ++++-- 2 files changed, 83 insertions(+), 60 deletions(-) diff --git a/doc/source/workers.rst b/doc/source/workers.rst index f02caca7..e54d38eb 100644 --- a/doc/source/workers.rst +++ b/doc/source/workers.rst @@ -31,11 +31,12 @@ Executor these requests can be accepted and processed by remote workers. Worker - Workers are started on remote hosts and has list of tasks it can perform (on - request). Workers accept and process task requests that are published by an - executor. Several requests can be processed simultaneously in separate - threads. For example, an `executor`_ can be passed to the worker and - configured to run in as many threads (green or not) as desired. + Workers are started on remote hosts and each has a list of tasks it can + perform (on request). Workers accept and process task requests that are + published by an executor. Several requests can be processed simultaneously + in separate threads (or processes...). For example, an `executor`_ can be + passed to the worker and configured to run in as many threads (green or + not) as desired. Proxy Executors interact with workers via a proxy. The proxy maintains the @@ -153,8 +154,16 @@ engine executor in the following manner: from dicts after receiving on both executor & worker sides (this translation is lossy since the traceback won't be fully retained). -Executor execute format -~~~~~~~~~~~~~~~~~~~~~~~ +Protocol +~~~~~~~~ + +.. automodule:: taskflow.engines.worker_based.protocol + +Examples +~~~~~~~~ + +Request (execute) +""""""""""""""""" * **task_name** - full task name to be performed * **task_cls** - full task class name to be performed @@ -186,8 +195,52 @@ Additionally, the following parameters are added to the request message: ] } -Worker response format -~~~~~~~~~~~~~~~~~~~~~~ + +Request (revert) +"""""""""""""""" + +When **reverting:** + +.. code:: json + + { + "action": "revert", + "arguments": {}, + "failures": { + "taskflow.tests.utils.TaskWithFailure": { + "exc_type_names": [ + "RuntimeError", + "StandardError", + "Exception" + ], + "exception_str": "Woot!", + "traceback_str": " File \"/homes/harlowja/dev/os/taskflow/taskflow/engines/action_engine/executor.py\", line 56, in _execute_task\n result = task.execute(**arguments)\n File \"/homes/harlowja/dev/os/taskflow/taskflow/tests/utils.py\", line 165, in execute\n raise RuntimeError('Woot!')\n", + "version": 1 + } + }, + "result": [ + "failure", + { + "exc_type_names": [ + "RuntimeError", + "StandardError", + "Exception" + ], + "exception_str": "Woot!", + "traceback_str": " File \"/homes/harlowja/dev/os/taskflow/taskflow/engines/action_engine/executor.py\", line 56, in _execute_task\n result = task.execute(**arguments)\n File \"/homes/harlowja/dev/os/taskflow/taskflow/tests/utils.py\", line 165, in execute\n raise RuntimeError('Woot!')\n", + "version": 1 + } + ], + "task_cls": "taskflow.tests.utils.TaskWithFailure", + "task_name": "taskflow.tests.utils.TaskWithFailure", + "task_version": [ + 1, + 0 + ] + } + +Worker response(s) +"""""""""""""""""" When **running:** @@ -241,49 +294,6 @@ When **failed:** "state": "FAILURE" } -Executor revert format -~~~~~~~~~~~~~~~~~~~~~~ - -When **reverting:** - -.. code:: json - - { - "action": "revert", - "arguments": {}, - "failures": { - "taskflow.tests.utils.TaskWithFailure": { - "exc_type_names": [ - "RuntimeError", - "StandardError", - "Exception" - ], - "exception_str": "Woot!", - "traceback_str": " File \"/homes/harlowja/dev/os/taskflow/taskflow/engines/action_engine/executor.py\", line 56, in _execute_task\n result = task.execute(**arguments)\n File \"/homes/harlowja/dev/os/taskflow/taskflow/tests/utils.py\", line 165, in execute\n raise RuntimeError('Woot!')\n", - "version": 1 - } - }, - "result": [ - "failure", - { - "exc_type_names": [ - "RuntimeError", - "StandardError", - "Exception" - ], - "exception_str": "Woot!", - "traceback_str": " File \"/homes/harlowja/dev/os/taskflow/taskflow/engines/action_engine/executor.py\", line 56, in _execute_task\n result = task.execute(**arguments)\n File \"/homes/harlowja/dev/os/taskflow/taskflow/tests/utils.py\", line 165, in execute\n raise RuntimeError('Woot!')\n", - "version": 1 - } - ], - "task_cls": "taskflow.tests.utils.TaskWithFailure", - "task_name": "taskflow.tests.utils.TaskWithFailure", - "task_version": [ - 1, - 0 - ] - } - Request state transitions ------------------------- diff --git a/taskflow/engines/worker_based/protocol.py b/taskflow/engines/worker_based/protocol.py index 19813d40..0dd79654 100644 --- a/taskflow/engines/worker_based/protocol.py +++ b/taskflow/engines/worker_based/protocol.py @@ -121,12 +121,16 @@ class Message(object): class Notify(Message): """Represents notify message type.""" + + #: String constant representing this message type. TYPE = NOTIFY # NOTE(harlowja): the executor (the entity who initially requests a worker # to send back a notification response) schema is different than the # worker response schema (that's why there are two schemas here). - _RESPONSE_SCHEMA = { + + #: Expected notify *response* message schema (in json schema format). + RESPONSE_SCHEMA = { "type": "object", 'properties': { 'topic': { @@ -142,7 +146,9 @@ class Notify(Message): "required": ["topic", 'tasks'], "additionalProperties": False, } - _SENDER_SCHEMA = { + + #: Expected *sender* request message schema (in json schema format). + SENDER_SCHEMA = { "type": "object", "additionalProperties": False, } @@ -156,9 +162,9 @@ class Notify(Message): @classmethod def validate(cls, data, response): if response: - schema = cls._RESPONSE_SCHEMA + schema = cls.RESPONSE_SCHEMA else: - schema = cls._SENDER_SCHEMA + schema = cls.SENDER_SCHEMA try: jsonschema.validate(data, schema, types=_SCHEMA_TYPES) except schema_exc.ValidationError as e: @@ -180,8 +186,11 @@ class Request(Message): states. """ + #: String constant representing this message type. TYPE = REQUEST - _SCHEMA = { + + #: Expected message schema (in json schema format). + SCHEMA = { "type": "object", 'properties': { # These two are typically only sent on revert actions (that is @@ -346,7 +355,7 @@ class Request(Message): @classmethod def validate(cls, data): try: - jsonschema.validate(data, cls._SCHEMA, types=_SCHEMA_TYPES) + jsonschema.validate(data, cls.SCHEMA, types=_SCHEMA_TYPES) except schema_exc.ValidationError as e: raise excp.InvalidFormat("%s message response data not of the" " expected format: %s" @@ -355,8 +364,12 @@ class Request(Message): class Response(Message): """Represents response message type.""" + + #: String constant representing this message type. TYPE = RESPONSE - _SCHEMA = { + + #: Expected message schema (in json schema format). + SCHEMA = { "type": "object", 'properties': { 'state': { @@ -439,7 +452,7 @@ class Response(Message): @classmethod def validate(cls, data): try: - jsonschema.validate(data, cls._SCHEMA, types=_SCHEMA_TYPES) + jsonschema.validate(data, cls.SCHEMA, types=_SCHEMA_TYPES) except schema_exc.ValidationError as e: raise excp.InvalidFormat("%s message response data not of the" " expected format: %s" From fc9cb88228204c37c43daaae799146fa1bb29213 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 23 Jan 2015 00:41:58 -0800 Subject: [PATCH 219/240] Use explicit WBE worker object arguments (instead of kwargs) Removes the kwargs usage that is now uniform across the other WBE components from the workers module so that the usage of kwargs for setting up these objects no longer is valid. Change-Id: I4e25b88c5d2f7e2d7933ff270e2782cebe227025 --- taskflow/engines/worker_based/worker.py | 22 ++++++++++++------- .../tests/unit/worker_based/test_worker.py | 19 +++++++++++----- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/taskflow/engines/worker_based/worker.py b/taskflow/engines/worker_based/worker.py index 5ac0cf4f..4273af60 100644 --- a/taskflow/engines/worker_based/worker.py +++ b/taskflow/engines/worker_based/worker.py @@ -20,13 +20,13 @@ import socket import string import sys -from concurrent import futures from oslo_utils import reflection from taskflow.engines.worker_based import endpoint from taskflow.engines.worker_based import server from taskflow import logging from taskflow import task as t_task +from taskflow.types import futures from taskflow.utils import misc from taskflow.utils import threading_utils as tu from taskflow import version @@ -77,23 +77,26 @@ class Worker(object): will be used to create tasks from. :param executor: custom executor object that can used for processing requests in separate threads (if not provided one will be created) - :param threads_count: threads count to be passed to the default executor + :param threads_count: threads count to be passed to the + default executor (used only if an executor is not + passed in) :param transport: transport to be used (e.g. amqp, memory, etc.) :param transport_options: transport specific options :param retry_options: retry specific options (used to configure how kombu handles retrying under tolerable/transient failures). """ - def __init__(self, exchange, topic, tasks, executor=None, **kwargs): + def __init__(self, exchange, topic, tasks, + executor=None, threads_count=None, url=None, + transport=None, transport_options=None, + retry_options=None): self._topic = topic self._executor = executor self._owns_executor = False self._threads_count = -1 if self._executor is None: - if 'threads_count' in kwargs: - self._threads_count = int(kwargs.pop('threads_count')) - if self._threads_count <= 0: - raise ValueError("threads_count provided must be > 0") + if threads_count is not None: + self._threads_count = int(threads_count) else: self._threads_count = tu.get_optimal_thread_count() self._executor = futures.ThreadPoolExecutor(self._threads_count) @@ -101,7 +104,10 @@ class Worker(object): self._endpoints = self._derive_endpoints(tasks) self._exchange = exchange self._server = server.Server(topic, exchange, self._executor, - self._endpoints, **kwargs) + self._endpoints, url=url, + transport=transport, + transport_options=transport_options, + retry_options=retry_options) @staticmethod def _derive_endpoints(tasks): diff --git a/taskflow/tests/unit/worker_based/test_worker.py b/taskflow/tests/unit/worker_based/test_worker.py index a572beb4..cc4578c0 100644 --- a/taskflow/tests/unit/worker_based/test_worker.py +++ b/taskflow/tests/unit/worker_based/test_worker.py @@ -64,7 +64,11 @@ class TestWorker(test.MockTestCase): master_mock_calls = [ mock.call.executor_class(self.threads_count), mock.call.Server(self.topic, self.exchange, - self.executor_inst_mock, [], url=self.broker_url) + self.executor_inst_mock, [], + url=self.broker_url, + transport_options=mock.ANY, + transport=mock.ANY, + retry_options=mock.ANY) ] self.assertEqual(self.master_mock.mock_calls, master_mock_calls) @@ -83,20 +87,23 @@ class TestWorker(test.MockTestCase): mock.call.executor_class(10), mock.call.Server(self.topic, self.exchange, self.executor_inst_mock, [], - url=self.broker_url) + url=self.broker_url, + transport_options=mock.ANY, + transport=mock.ANY, + retry_options=mock.ANY) ] self.assertEqual(self.master_mock.mock_calls, master_mock_calls) - def test_creation_with_negative_threads_count(self): - self.assertRaises(ValueError, self.worker, threads_count=-10) - def test_creation_with_custom_executor(self): executor_mock = mock.MagicMock(name='executor') self.worker(executor=executor_mock) master_mock_calls = [ mock.call.Server(self.topic, self.exchange, executor_mock, [], - url=self.broker_url) + url=self.broker_url, + transport_options=mock.ANY, + transport=mock.ANY, + retry_options=mock.ANY) ] self.assertEqual(self.master_mock.mock_calls, master_mock_calls) From 35745c902eee8bbcf24da6fdd7139fc08513faeb Mon Sep 17 00:00:00 2001 From: Ivan Melnikov Date: Fri, 23 Jan 2015 14:14:53 +0300 Subject: [PATCH 220/240] Fix coverage environment To run coverage report we need 'coverage' package, which should be added to dependency list for corresponding tox environment. Change-Id: I546b5afb1501773351fc2233bb12858497970105 --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index a295abfd..4ef1c335 100644 --- a/tox.ini +++ b/tox.ini @@ -40,6 +40,7 @@ commands = pylint --rcfile=pylintrc taskflow [testenv:cover] basepython = python2.7 deps = {[testenv:py27]deps} + coverage>=3.6 commands = python setup.py testr --coverage --testr-args='{posargs}' [testenv:venv] From 66fc2df3a61198df8d9b3acf1ac9fed6b87ae060 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 23 Jan 2015 14:45:24 -0800 Subject: [PATCH 221/240] Add comments to runner state machine reaction functions To make it a little more obvious what each function is responsible for doing (and why) add comments onto each inner reaction function that explains more of its purpose in life. Change-Id: Ibdb55e873e4b1ae5675e2f50d85ac0ecbd59ffb6 --- taskflow/engines/action_engine/runner.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/taskflow/engines/action_engine/runner.py b/taskflow/engines/action_engine/runner.py index a5bebd61..f1f880ce 100644 --- a/taskflow/engines/action_engine/runner.py +++ b/taskflow/engines/action_engine/runner.py @@ -99,11 +99,19 @@ class _MachineBuilder(object): timeout = _WAITING_TIMEOUT def resume(old_state, new_state, event): + # This reaction function just updates the state machines memory + # to include any nodes that need to be executed (from a previous + # attempt, which may be empty if never ran before) and any nodes + # that are now ready to be ran. memory.next_nodes.update(self._completer.resume()) memory.next_nodes.update(self._analyzer.get_next_nodes()) return _SCHEDULE def game_over(old_state, new_state, event): + # This reaction function is mainly a intermediary delegation + # function that analyzes the current memory and transitions to + # the appropriate handler that will deal with the memory values, + # it is *always* called before the final state is entered. if memory.failures: return _FAILED if self._analyzer.get_next_nodes(): @@ -114,6 +122,11 @@ class _MachineBuilder(object): return _REVERTED def schedule(old_state, new_state, event): + # This reaction function starts to schedule the memory's next + # nodes (iff the engine is still runnable, which it may not be + # if the user of this engine has requested the engine/storage + # that holds this information to stop or suspend); handles failures + # that occur during this process safely... if self.runnable() and memory.next_nodes: not_done, failures = self._scheduler.schedule( memory.next_nodes) @@ -136,6 +149,11 @@ class _MachineBuilder(object): return _ANALYZE def analyze(old_state, new_state, event): + # This reaction function is responsible for analyzing all nodes + # that have finished executing and completing them and figuring + # out what nodes are now ready to be ran (and then triggering those + # nodes to be scheduled in the future); handles failures that + # occur during this process safely... next_nodes = set() while memory.done: fut = memory.done.pop() From e3e2950bf5efbbfb4358e5a5f0dd32932400c30a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 23 Jan 2015 14:21:59 -0800 Subject: [PATCH 222/240] Allow just specifying 'workers' for WBE entrypoint Instead of requiring and only finding 'worker-based' just instead add the ability to use 'workers' as the addition of '-based' doesn't add much value and isn't really that beneficial/useful to have to type/require. Change-Id: Icacb933e87147e421589457338d13fd8d59e0e55 --- doc/source/engines.rst | 6 +++--- setup.cfg | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/source/engines.rst b/doc/source/engines.rst index d975825c..c9b0c5da 100644 --- a/doc/source/engines.rst +++ b/doc/source/engines.rst @@ -182,10 +182,10 @@ using your desired execution model. :pybug:`22393` and others...) as the most recent python version (which themselves have a variety of ongoing/recent bugs). -Worker-based ------------- +Workers +------- -**Engine type**: ``'worker-based'`` +**Engine type**: ``'worker-based'`` or ``'workers'`` .. note:: Since this engine is significantly more complicated (and different) then the others we thought it appropriate to devote a diff --git a/setup.cfg b/setup.cfg index cfef68eb..fcaff44d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,6 +51,7 @@ taskflow.engines = serial = taskflow.engines.action_engine.engine:SerialActionEngine parallel = taskflow.engines.action_engine.engine:ParallelActionEngine worker-based = taskflow.engines.worker_based.engine:WorkerBasedActionEngine + workers = taskflow.engines.worker_based.engine:WorkerBasedActionEngine [nosetests] cover-erase = true From 2f04395b96554b41f2d3ebed4d5f26ba2ff0a0a8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 23 Jan 2015 15:03:21 -0800 Subject: [PATCH 223/240] Leave use-cases out of WBE developer documentation It doesn't seem appropriate anymore to include the use-cases for the worker engine in developer documentation since that information should be placed elsewhere (as its not especially relevant to developers). Change-Id: I7eae79bd630572d0c045a2561f2f36bdcf14aaf4 --- doc/source/workers.rst | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/doc/source/workers.rst b/doc/source/workers.rst index e54d38eb..df7e4b63 100644 --- a/doc/source/workers.rst +++ b/doc/source/workers.rst @@ -68,29 +68,6 @@ Requirements .. _executor: https://docs.python.org/dev/library/concurrent.futures.html#executor-objects .. _protocol: http://en.wikipedia.org/wiki/Communications_protocol -Use-cases ---------- - -* `Glance`_ - - * Image tasks *(long-running)* - - * Convert, import/export & more... - -* `Heat`_ - - * Engine work distribution - -* `Rally`_ - - * Load generation - -* *Your use-case here* - -.. _Heat: https://wiki.openstack.org/wiki/Heat -.. _Rally: https://wiki.openstack.org/wiki/Rally -.. _Glance: https://wiki.openstack.org/wiki/Glance - Design ====== From e417914d49458a4f0fd231497f6a4c1200cfecb2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 23 Jan 2015 18:05:02 -0800 Subject: [PATCH 224/240] Make all/most usage of type errors follow a similar pattern To make it easy to add new type errors and to make the existing ones have a common string pattern adjust the current type errors to contain at least the following string format: '%s' (%s) where these two places will be filled in with the object of the wrong type and the type of that object. This information is useful when analyzing the exception (by the user) to know exactly what they passed in and what type it was. This convention is not maintained where it would interpolate large text blobs (such as in binary encoding/decoding and json decoding). Change-Id: Id84b0e7ce684a543cc407b15016e77804e6f03ed --- taskflow/atom.py | 3 +- taskflow/engines/action_engine/engine.py | 5 +-- taskflow/engines/action_engine/scheduler.py | 3 +- taskflow/persistence/backends/impl_memory.py | 4 +-- taskflow/persistence/base.py | 4 +-- taskflow/persistence/logbook.py | 5 +-- taskflow/storage.py | 4 +-- taskflow/tests/unit/test_task.py | 2 +- taskflow/types/failure.py | 3 +- taskflow/utils/kazoo_utils.py | 3 +- taskflow/utils/misc.py | 33 ++++++++++++-------- 11 files changed, 41 insertions(+), 28 deletions(-) diff --git a/taskflow/atom.py b/taskflow/atom.py index a2066048..d236ff90 100644 --- a/taskflow/atom.py +++ b/taskflow/atom.py @@ -69,7 +69,8 @@ def _build_rebind_dict(args, rebind_args): elif isinstance(rebind_args, dict): return rebind_args else: - raise TypeError('Invalid rebind value: %s' % rebind_args) + raise TypeError("Invalid rebind value '%s' (%s)" + % (rebind_args, type(rebind_args))) def _build_arg_mapping(atom_name, reqs, rebind_args, function, do_infer, diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py index ed06b64f..51df5698 100644 --- a/taskflow/engines/action_engine/engine.py +++ b/taskflow/engines/action_engine/engine.py @@ -344,8 +344,9 @@ String (case insensitive) Executor used expected = set() for m in cls._executor_cls_matchers: expected.update(m.types) - raise TypeError("Unknown executor type '%s' expected an" - " instance of %s" % (type(desired_executor), + raise TypeError("Unknown executor '%s' (%s) expected an" + " instance of %s" % (desired_executor, + type(desired_executor), list(expected))) else: executor_cls = matched_executor_cls diff --git a/taskflow/engines/action_engine/scheduler.py b/taskflow/engines/action_engine/scheduler.py index 82bacbe4..8e3c64b3 100644 --- a/taskflow/engines/action_engine/scheduler.py +++ b/taskflow/engines/action_engine/scheduler.py @@ -91,7 +91,8 @@ class Scheduler(object): if sched.handles(node): return sched.schedule(node) else: - raise TypeError("Unknown how to schedule '%s'" % node) + raise TypeError("Unknown how to schedule '%s' (%s)" + % (node, type(node))) def schedule(self, nodes): """Schedules the provided nodes for *future* completion. diff --git a/taskflow/persistence/backends/impl_memory.py b/taskflow/persistence/backends/impl_memory.py index 2f8c4f68..5e94afb1 100644 --- a/taskflow/persistence/backends/impl_memory.py +++ b/taskflow/persistence/backends/impl_memory.py @@ -82,8 +82,8 @@ class _MemoryHelper(object): saved_info = self._memory.atom_details.setdefault( incoming.uuid, {}) else: - raise TypeError("Unknown how to merge type '%s'" - % type(incoming)) + raise TypeError("Unknown how to merge '%s' (%s)" + % (incoming, type(incoming))) try: saved_info['object'].merge(incoming) except KeyError: diff --git a/taskflow/persistence/base.py b/taskflow/persistence/base.py index 9185d69c..00fb29be 100644 --- a/taskflow/persistence/base.py +++ b/taskflow/persistence/base.py @@ -29,8 +29,8 @@ class Backend(object): if not conf: conf = {} if not isinstance(conf, dict): - raise TypeError("Configuration dictionary expected not: %s" - % type(conf)) + raise TypeError("Configuration dictionary expected not '%s' (%s)" + % (conf, type(conf))) self._conf = conf @abc.abstractmethod diff --git a/taskflow/persistence/logbook.py b/taskflow/persistence/logbook.py index 9ef3efec..fcd777da 100644 --- a/taskflow/persistence/logbook.py +++ b/taskflow/persistence/logbook.py @@ -583,11 +583,12 @@ def atom_detail_class(atom_type): try: return _NAME_TO_DETAIL[atom_type] except KeyError: - raise TypeError("Unknown atom type: %s" % (atom_type)) + raise TypeError("Unknown atom type '%s'" % (atom_type)) def atom_detail_type(atom_detail): try: return _DETAIL_TO_NAME[type(atom_detail)] except KeyError: - raise TypeError("Unknown atom type: %s" % type(atom_detail)) + raise TypeError("Unknown atom '%s' (%s)" + % (atom_detail, type(atom_detail))) diff --git a/taskflow/storage.py b/taskflow/storage.py index 8cb81f3b..df801483 100644 --- a/taskflow/storage.py +++ b/taskflow/storage.py @@ -183,8 +183,8 @@ class Storage(object): misc.get_version_string(atom), atom.save_as) else: - raise TypeError("Object of type 'atom' expected." - " Got %s, %r." % (type(atom), atom)) + raise TypeError("Object of type 'atom' expected not" + " '%s' (%s)" % (atom, type(atom))) def _ensure_task(self, task_name, task_version, result_mapping): """Ensures there is a taskdetail that corresponds to the task info. diff --git a/taskflow/tests/unit/test_task.py b/taskflow/tests/unit/test_task.py index 2f415840..50e783f3 100644 --- a/taskflow/tests/unit/test_task.py +++ b/taskflow/tests/unit/test_task.py @@ -181,7 +181,7 @@ class TaskTest(test.TestCase): }) def test_rebind_list_bad_value(self): - self.assertRaisesRegexp(TypeError, '^Invalid rebind value:', + self.assertRaisesRegexp(TypeError, '^Invalid rebind value', MyTask, rebind=object()) def test_default_provides(self): diff --git a/taskflow/types/failure.py b/taskflow/types/failure.py index 4732fb14..0f45bc35 100644 --- a/taskflow/types/failure.py +++ b/taskflow/types/failure.py @@ -147,7 +147,8 @@ class Failure(object): self._exc_type_names = tuple( reflection.get_all_class_names(exc_info[0], up_to=Exception)) if not self._exc_type_names: - raise TypeError('Invalid exception type: %r' % exc_info[0]) + raise TypeError("Invalid exception type '%s' (%s)" + % (exc_info[0], type(exc_info[0]))) self._exception_str = exc.exception_message(self._exc_info[1]) self._traceback_str = ''.join( traceback.format_tb(self._exc_info[2])) diff --git a/taskflow/utils/kazoo_utils.py b/taskflow/utils/kazoo_utils.py index 35fe058e..c2869bdd 100644 --- a/taskflow/utils/kazoo_utils.py +++ b/taskflow/utils/kazoo_utils.py @@ -185,7 +185,8 @@ def make_client(conf): hosts = _parse_hosts(conf.get("hosts", "localhost:2181")) if not hosts or not isinstance(hosts, six.string_types): raise TypeError("Invalid hosts format, expected " - "non-empty string/list, not %s" % type(hosts)) + "non-empty string/list, not '%s' (%s)" + % (hosts, type(hosts))) client_kwargs['hosts'] = hosts if 'timeout' in conf: client_kwargs['timeout'] = float(conf['timeout']) diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 7a196512..8905906a 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -92,15 +92,15 @@ def find_subclasses(locations, base_cls, exclude_hidden=True): else: obj = importutils.import_class('%s.%s' % (pkg, cls)) if not reflection.is_subclass(obj, base_cls): - raise TypeError("Item %s is not a %s subclass" % - (item, base_cls)) + raise TypeError("Object '%s' (%s) is not a '%s' subclass" + % (item, type(item), base_cls)) derived.add(obj) elif isinstance(item, types.ModuleType): module = item elif reflection.is_subclass(item, base_cls): derived.add(item) else: - raise TypeError("Item %s unexpected type: %s" % + raise TypeError("Object '%s' (%s) is an unexpected type" % (item, type(item))) # If it's a module derive objects from it if we can. if module is not None: @@ -125,11 +125,10 @@ def parse_uri(uri): # Do some basic validation before continuing... if not isinstance(uri, six.string_types): raise TypeError("Can only parse string types to uri data, " - "and not an object of type %s" - % reflection.get_class_name(uri)) + "and not '%s' (%s)" % (uri, type(uri))) match = _SCHEME_REGEX.match(uri) if not match: - raise ValueError("Uri %r does not start with a RFC 3986 compliant" + raise ValueError("Uri '%s' does not start with a RFC 3986 compliant" " scheme" % (uri)) return netutils.urlsplit(uri) @@ -165,7 +164,7 @@ def binary_encode(text, encoding='utf-8'): elif isinstance(text, six.text_type): return text.encode(encoding) else: - raise TypeError("Expected binary or string type") + raise TypeError("Expected binary or string type not '%s'" % type(text)) def binary_decode(data, encoding='utf-8'): @@ -178,7 +177,7 @@ def binary_decode(data, encoding='utf-8'): elif isinstance(data, six.text_type): return data else: - raise TypeError("Expected binary or string type") + raise TypeError("Expected binary or string type not '%s'" % type(data)) def decode_json(raw_data, root_types=(dict,)): @@ -194,10 +193,17 @@ def decode_json(raw_data, root_types=(dict,)): raise ValueError("Expected UTF-8 decodable data: %s" % e) except ValueError as e: raise ValueError("Expected JSON decodable data: %s" % e) - if root_types and not isinstance(data, tuple(root_types)): - ok_types = ", ".join(str(t) for t in root_types) - raise ValueError("Expected (%s) root types not: %s" - % (ok_types, type(data))) + if root_types: + if not isinstance(root_types, tuple): + root_types = tuple(root_types) + if not isinstance(data, root_types): + if len(root_types) == 1: + root_type = root_types[0] + raise ValueError("Expected '%s' root type not '%s'" + % (root_type, type(data))) + else: + raise ValueError("Expected %s root types not '%s'" + % (list(root_types), type(data))) return data @@ -346,7 +352,8 @@ def as_int(obj, quiet=False): pass # Eck, not sure what this is then. if not quiet: - raise TypeError("Can not translate %s to an integer." % (obj)) + raise TypeError("Can not translate '%s' (%s) to an integer" + % (obj, type(obj))) return obj From ca82e20efe8f5c5d50b3db89be0342710ef7f73b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 24 Jan 2015 00:45:36 -0800 Subject: [PATCH 225/240] Add a thread bundle helper utility + tests To make it easier to create a bunch of threads in a single call (and stop them in a single call) create a concept of a thread bundle (similar to a thread group) that will call into a provided set of factories to get a thread, activate callbacks to notify others that a thread is about to start or stop and then perform the start or stop of the bound threads in a orderly manner. Change-Id: I7d233cccb230b716af41243ad27220b988eec14c --- taskflow/engines/worker_based/executor.py | 28 ++--- .../tests/unit/test_utils_threading_utils.py | 115 ++++++++++++++++++ .../tests/unit/worker_based/test_executor.py | 3 - taskflow/utils/threading_utils.py | 104 ++++++++++++++++ 4 files changed, 229 insertions(+), 21 deletions(-) create mode 100644 taskflow/tests/unit/test_utils_threading_utils.py diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index cda37458..8290ba61 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -59,10 +59,16 @@ class WorkerTaskExecutor(executor.TaskExecutor): transport=transport, transport_options=transport_options, retry_options=retry_options) - self._proxy_thread = None self._periodic = wt.PeriodicWorker(tt.Timeout(pr.NOTIFY_PERIOD), [self._notify_topics]) - self._periodic_thread = None + self._helpers = tu.ThreadBundle() + self._helpers.bind(lambda: tu.daemon_thread(self._proxy.start), + after_start=lambda t: self._proxy.wait(), + before_join=lambda t: self._proxy.stop()) + self._helpers.bind(lambda: tu.daemon_thread(self._periodic.start), + before_join=lambda t: self._periodic.stop(), + after_join=lambda t: self._periodic.reset(), + before_start=lambda t: self._periodic.reset()) def _process_notify(self, notify, message): """Process notify message from remote side.""" @@ -226,24 +232,10 @@ class WorkerTaskExecutor(executor.TaskExecutor): def start(self): """Starts proxy thread and associated topic notification thread.""" - if not tu.is_alive(self._proxy_thread): - self._proxy_thread = tu.daemon_thread(self._proxy.start) - self._proxy_thread.start() - self._proxy.wait() - if not tu.is_alive(self._periodic_thread): - self._periodic.reset() - self._periodic_thread = tu.daemon_thread(self._periodic.start) - self._periodic_thread.start() + self._helpers.start() def stop(self): """Stops proxy thread and associated topic notification thread.""" - if self._periodic_thread is not None: - self._periodic.stop() - self._periodic_thread.join() - self._periodic_thread = None - if self._proxy_thread is not None: - self._proxy.stop() - self._proxy_thread.join() - self._proxy_thread = None + self._helpers.stop() self._requests_cache.clear(self._handle_expired_request) self._workers.clear() diff --git a/taskflow/tests/unit/test_utils_threading_utils.py b/taskflow/tests/unit/test_utils_threading_utils.py new file mode 100644 index 00000000..974285fa --- /dev/null +++ b/taskflow/tests/unit/test_utils_threading_utils.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +import time + +from taskflow import test +from taskflow.utils import threading_utils as tu + + +def _spinner(death): + while not death.is_set(): + time.sleep(0.1) + + +class TestThreadHelpers(test.TestCase): + def test_event_wait(self): + e = tu.Event() + e.set() + self.assertTrue(e.wait()) + + def test_alive_thread_falsey(self): + for v in [False, 0, None, ""]: + self.assertFalse(tu.is_alive(v)) + + def test_alive_thread(self): + death = tu.Event() + t = tu.daemon_thread(_spinner, death) + self.assertFalse(tu.is_alive(t)) + t.start() + self.assertTrue(tu.is_alive(t)) + death.set() + t.join() + self.assertFalse(tu.is_alive(t)) + + def test_daemon_thread(self): + death = tu.Event() + t = tu.daemon_thread(_spinner, death) + self.assertTrue(t.daemon) + + +class TestThreadBundle(test.TestCase): + thread_count = 5 + + def setUp(self): + super(TestThreadBundle, self).setUp() + self.bundle = tu.ThreadBundle() + self.death = tu.Event() + self.addCleanup(self.bundle.stop) + self.addCleanup(self.death.set) + + def test_bind_invalid(self): + self.assertRaises(ValueError, self.bundle.bind, 1) + for k in ['after_start', 'before_start', + 'before_join', 'after_join']: + kwargs = { + k: 1, + } + self.assertRaises(ValueError, self.bundle.bind, + lambda: tu.daemon_thread(_spinner, self.death), + **kwargs) + + def test_bundle_length(self): + self.assertEqual(0, len(self.bundle)) + for i in range(0, self.thread_count): + self.bundle.bind(lambda: tu.daemon_thread(_spinner, self.death)) + self.assertEqual(1, self.bundle.start()) + self.assertEqual(i + 1, len(self.bundle)) + self.death.set() + self.assertEqual(self.thread_count, self.bundle.stop()) + self.assertEqual(self.thread_count, len(self.bundle)) + + def test_start_stop(self): + events = collections.deque() + + def before_start(t): + events.append('bs') + + def before_join(t): + events.append('bj') + self.death.set() + + def after_start(t): + events.append('as') + + def after_join(t): + events.append('aj') + + for _i in range(0, self.thread_count): + self.bundle.bind(lambda: tu.daemon_thread(_spinner, self.death), + before_join=before_join, + after_join=after_join, + before_start=before_start, + after_start=after_start) + self.assertEqual(self.thread_count, self.bundle.start()) + self.assertEqual(self.thread_count, len(self.bundle)) + self.assertEqual(self.thread_count, self.bundle.stop()) + for event in ['as', 'bs', 'bj', 'aj']: + self.assertEqual(self.thread_count, + len([e for e in events if e == event])) + self.assertEqual(0, self.bundle.stop()) + self.assertTrue(self.death.is_set()) diff --git a/taskflow/tests/unit/worker_based/test_executor.py b/taskflow/tests/unit/worker_based/test_executor.py index cdb421d1..101031c4 100644 --- a/taskflow/tests/unit/worker_based/test_executor.py +++ b/taskflow/tests/unit/worker_based/test_executor.py @@ -353,9 +353,6 @@ class TestWorkerTaskExecutor(test.MockTestCase): ex = self.executor() ex.start() - # wait until executor thread is done - ex._proxy_thread.join() - # stop executor ex.stop() diff --git a/taskflow/utils/threading_utils.py b/taskflow/utils/threading_utils.py index 5048401c..cea0760d 100644 --- a/taskflow/utils/threading_utils.py +++ b/taskflow/utils/threading_utils.py @@ -14,10 +14,12 @@ # License for the specific language governing permissions and limitations # under the License. +import collections import multiprocessing import sys import threading +import six from six.moves import _thread @@ -71,3 +73,105 @@ def daemon_thread(target, *args, **kwargs): # unless the daemon property is set to True. thread.daemon = True return thread + + +# Container for thread creator + associated callbacks. +_ThreadBuilder = collections.namedtuple('_ThreadBuilder', + ['thread_factory', + 'before_start', 'after_start', + 'before_join', 'after_join']) +_ThreadBuilder.callables = tuple([ + # Attribute name -> none allowed as a valid value... + ('thread_factory', False), + ('before_start', True), + ('after_start', True), + ('before_join', True), + ('after_join', True), +]) + + +class ThreadBundle(object): + """A group/bundle of threads that start/stop together.""" + + def __init__(self): + self._threads = [] + self._lock = threading.Lock() + + def bind(self, thread_factory, + before_start=None, after_start=None, + before_join=None, after_join=None): + """Adds a thread (to-be) into this bundle (with given callbacks). + + NOTE(harlowja): callbacks provided should not attempt to call + mutating methods (:meth:`.stop`, :meth:`.start`, + :meth:`.bind` ...) on this object as that will result + in dead-lock since the lock on this object is not + meant to be (and is not) reentrant... + """ + builder = _ThreadBuilder(thread_factory, + before_start, after_start, + before_join, after_join) + for attr_name, none_allowed in builder.callables: + cb = getattr(builder, attr_name) + if cb is None and none_allowed: + continue + if not six.callable(cb): + raise ValueError("Provided callback for argument" + " '%s' must be callable" % attr_name) + with self._lock: + self._threads.append([ + builder, + # The built thread. + None, + # Whether the built thread was started (and should have + # ran or still be running). + False, + ]) + + @staticmethod + def _trigger_callback(callback, thread): + if callback is not None: + callback(thread) + + def start(self): + """Creates & starts all associated threads (that are not running).""" + count = 0 + with self._lock: + for i, (builder, thread, started) in enumerate(self._threads): + if thread and started: + continue + if not thread: + self._threads[i][1] = thread = builder.thread_factory() + self._trigger_callback(builder.before_start, thread) + thread.start() + count += 1 + try: + self._trigger_callback(builder.after_start, thread) + finally: + # Just incase the 'after_start' callback blows up make sure + # we always set this... + self._threads[i][2] = started = True + return count + + def stop(self): + """Stops & joins all associated threads (that have been started).""" + count = 0 + with self._lock: + for i, (builder, thread, started) in enumerate(self._threads): + if not thread or not started: + continue + self._trigger_callback(builder.before_join, thread) + thread.join() + count += 1 + try: + self._trigger_callback(builder.after_join, thread) + finally: + # Just incase the 'after_join' callback blows up make sure + # we always set/reset these... + self._threads[i][1] = thread = None + self._threads[i][2] = started = False + return count + + def __len__(self): + """Returns how many threads (to-be) are in this bundle.""" + return len(self._threads) From 84d44fad112a25658a9ab8ff2918c048bcbf44e3 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 25 Jan 2015 10:48:12 -0800 Subject: [PATCH 226/240] Shrink the WBE request transition SVG image size Since this image/svg is not as dense we don't need to use the same size (660px) as the other state diagrams. Using a smaller size (520px) looks nicer and fits better on the documentation page for this less dense diagram. Change-Id: I803ec7d83096da4f4b45c2c1858b82f2df9d38f9 --- doc/source/workers.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/workers.rst b/doc/source/workers.rst index df7e4b63..86548992 100644 --- a/doc/source/workers.rst +++ b/doc/source/workers.rst @@ -275,8 +275,8 @@ Request state transitions ------------------------- .. image:: img/wbe_request_states.svg - :width: 660px - :align: left + :width: 520px + :align: center :alt: WBE request state transitions **WAITING** - Request placed on queue (or other `kombu`_ message bus/transport) From 97797abb518f6b73f4165206bac73e770f4d1aa3 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 25 Jan 2015 11:08:57 -0800 Subject: [PATCH 227/240] Use importutils.try_import for optional eventlet imports Instead of doing try: except ImportError for eventlet imports use the oslo.utils importutils.try_import function that safely does the same with less verbosity for the optional usage of eventlet imports/code that we have. Change-Id: I7eaa7c5908ffb04282892c9f6af04044b73f4f8c --- taskflow/types/futures.py | 12 +++++------- taskflow/utils/async_utils.py | 6 ++---- taskflow/utils/eventlet_utils.py | 10 +++++----- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/taskflow/types/futures.py b/taskflow/types/futures.py index 128f9aa5..1d847ddc 100644 --- a/taskflow/types/futures.py +++ b/taskflow/types/futures.py @@ -20,15 +20,13 @@ import threading from concurrent import futures as _futures from concurrent.futures import process as _process from concurrent.futures import thread as _thread +from oslo_utils import importutils from oslo_utils import reflection -try: - from eventlet.green import threading as greenthreading - from eventlet import greenpool - from eventlet import patcher as greenpatcher - from eventlet import queue as greenqueue -except ImportError: - pass +greenpatcher = importutils.try_import('eventlet.patcher') +greenpool = importutils.try_import('eventlet.greenpool') +greenqueue = importutils.try_import('eventlet.queue') +greenthreading = importutils.try_import('eventlet.green.threading') from taskflow.types import timing from taskflow.utils import eventlet_utils as eu diff --git a/taskflow/utils/async_utils.py b/taskflow/utils/async_utils.py index 71eafa18..e04c44e7 100644 --- a/taskflow/utils/async_utils.py +++ b/taskflow/utils/async_utils.py @@ -16,11 +16,9 @@ from concurrent import futures as _futures from concurrent.futures import _base +from oslo_utils import importutils -try: - from eventlet.green import threading as greenthreading -except ImportError: - pass +greenthreading = importutils.try_import('eventlet.green.threading') from taskflow.types import futures from taskflow.utils import eventlet_utils as eu diff --git a/taskflow/utils/eventlet_utils.py b/taskflow/utils/eventlet_utils.py index 68dd355a..2f5a42a6 100644 --- a/taskflow/utils/eventlet_utils.py +++ b/taskflow/utils/eventlet_utils.py @@ -14,11 +14,11 @@ # License for the specific language governing permissions and limitations # under the License. -try: - import eventlet as _eventlet # noqa - EVENTLET_AVAILABLE = True -except ImportError: - EVENTLET_AVAILABLE = False +from oslo_utils import importutils + +_eventlet = importutils.try_import('eventlet') + +EVENTLET_AVAILABLE = bool(_eventlet) def check_for_eventlet(exc=None): From 802bce9c53c0f96375bf2ff53dae59601e3aa0a7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 25 Jan 2015 11:57:46 -0800 Subject: [PATCH 228/240] Center SVG state diagrams Change-Id: I0a8fce9b619039f5d7316226847ac1d2fdd41d24 --- doc/source/states.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/states.rst b/doc/source/states.rst index 805c10c4..bba8d203 100644 --- a/doc/source/states.rst +++ b/doc/source/states.rst @@ -18,7 +18,7 @@ Engine .. image:: img/engine_states.svg :width: 660px - :align: left + :align: center :alt: Action engine state transitions **RESUMING** - Prepares flow & atoms to be resumed. @@ -47,7 +47,7 @@ Flow .. image:: img/flow_states.svg :width: 660px - :align: left + :align: center :alt: Flow state transitions **PENDING** - A flow starts its execution lifecycle in this state (it has no @@ -110,7 +110,7 @@ Task .. image:: img/task_states.svg :width: 660px - :align: left + :align: center :alt: Task state transitions **PENDING** - A task starts its execution lifecycle in this state (it has no @@ -154,7 +154,7 @@ Retry .. image:: img/retry_states.svg :width: 660px - :align: left + :align: center :alt: Retry state transitions **PENDING** - A retry starts its execution lifecycle in this state (it has no From f3a1dcb0a737a90f1c4728a610223868377d7484 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 26 Jan 2015 08:47:31 -0800 Subject: [PATCH 229/240] Emit a warning when no routing keys provided on publish() When no routing keys are provided the message will no be published anywhere; so when this happens emit a warning to tell the library user that something is likely misconfigured and/or wrong. Change-Id: Id27e8ab73c1ae8a02cb4252958d9dc896a5df1f0 --- taskflow/engines/worker_based/proxy.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/taskflow/engines/worker_based/proxy.py b/taskflow/engines/worker_based/proxy.py index 1770d562..160430a8 100644 --- a/taskflow/engines/worker_based/proxy.py +++ b/taskflow/engines/worker_based/proxy.py @@ -150,6 +150,14 @@ class Proxy(object): else: routing_keys = routing_key + # Filter out any empty keys... + routing_keys = [r_k for r_k in routing_keys if r_k] + if not routing_keys: + LOG.warn("No routing key/s specified; unable to send '%s'" + " to any target queue on exchange '%s'", msg, + self._exchange_name) + return + def _publish(producer, routing_key): queue = self._make_queue(routing_key, self._exchange) producer.publish(body=msg.to_dict(), From 7fe2945813450cd0ba1264171c5a90dc0f90d57f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 26 Jan 2015 16:26:56 -0800 Subject: [PATCH 230/240] Link WBE docs together better (especially around arguments) Link the engine/worker 'retry_options' to the proxy which has linkage to what the defaults are and to where more information can be gathered about the option (which really gets sent in to kombu) so that users can more easily understand it. Also removes showing the docs about the following as its more of an internal API/class not meant for public consumption: * Module: taskflow.engines.worker_based.executor Change-Id: Ib16ee10765ec6b9a0af320bd6818d9649c2485b1 --- doc/source/workers.rst | 1 - taskflow/engines/worker_based/engine.py | 8 +++++--- taskflow/engines/worker_based/proxy.py | 15 ++++++++------- taskflow/engines/worker_based/worker.py | 8 +++++--- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/doc/source/workers.rst b/doc/source/workers.rst index 86548992..7c4f0112 100644 --- a/doc/source/workers.rst +++ b/doc/source/workers.rst @@ -416,7 +416,6 @@ Interfaces ========== .. automodule:: taskflow.engines.worker_based.engine -.. automodule:: taskflow.engines.worker_based.executor .. automodule:: taskflow.engines.worker_based.proxy .. automodule:: taskflow.engines.worker_based.worker diff --git a/taskflow/engines/worker_based/engine.py b/taskflow/engines/worker_based/engine.py index a161ee58..aee39e89 100644 --- a/taskflow/engines/worker_based/engine.py +++ b/taskflow/engines/worker_based/engine.py @@ -32,7 +32,6 @@ class WorkerBasedActionEngine(engine.ActionEngine): be learned by listening to the notifications that workers emit). :param transport: transport to be used (e.g. amqp, memory, etc.) - :param transport_options: transport specific options :param transition_timeout: numeric value (or None for infinite) to wait for submitted remote requests to transition out of the (PENDING, WAITING) request states. When @@ -40,8 +39,11 @@ class WorkerBasedActionEngine(engine.ActionEngine): for will have its result become a `RequestTimeout` exception instead of its normally returned value (or raised exception). - :param retry_options: retry specific options (used to configure how kombu - handles retrying under tolerable/transient failures). + :param transport_options: transport specific options (see: + http://kombu.readthedocs.org/ for what these + options imply and are expected to be) + :param retry_options: retry specific options + (see: :py:attr:`~.proxy.Proxy.DEFAULT_RETRY_OPTIONS`) """ _storage_factory = t_storage.SingleThreadedStorage diff --git a/taskflow/engines/worker_based/proxy.py b/taskflow/engines/worker_based/proxy.py index 1770d562..59b9098a 100644 --- a/taskflow/engines/worker_based/proxy.py +++ b/taskflow/engines/worker_based/proxy.py @@ -44,12 +44,7 @@ _TransportDetails = collections.namedtuple('_TransportDetails', class Proxy(object): """A proxy processes messages from/to the named exchange.""" - # Settings that are by default used for consumers/producers to reconnect - # under tolerable/transient failures... - # - # See: http://kombu.readthedocs.org/en/latest/reference/kombu.html for - # what these values imply... - _DEFAULT_RETRY_OPTIONS = { + DEFAULT_RETRY_OPTIONS = { # The number of seconds we start sleeping for. 'interval_start': 1, # How many seconds added to the interval for each retry. @@ -59,6 +54,12 @@ class Proxy(object): # Maximum number of times to retry. 'max_retries': 3, } + """Settings used (by default) to reconnect under transient failures. + + See: http://kombu.readthedocs.org/ (and connection ``ensure_options``) for + what these values imply/mean... + """ + # This is the only provided option that should be an int, the others # are allowed to be floats; used when we check that the user-provided # value is valid... @@ -78,7 +79,7 @@ class Proxy(object): # running, otherwise requeue them. lambda data, message: not self.is_running) - ensure_options = self._DEFAULT_RETRY_OPTIONS.copy() + ensure_options = self.DEFAULT_RETRY_OPTIONS.copy() if retry_options is not None: # Override the defaults with any user provided values... for k in set(six.iterkeys(ensure_options)): diff --git a/taskflow/engines/worker_based/worker.py b/taskflow/engines/worker_based/worker.py index 4273af60..2110b92b 100644 --- a/taskflow/engines/worker_based/worker.py +++ b/taskflow/engines/worker_based/worker.py @@ -81,9 +81,11 @@ class Worker(object): default executor (used only if an executor is not passed in) :param transport: transport to be used (e.g. amqp, memory, etc.) - :param transport_options: transport specific options - :param retry_options: retry specific options (used to configure how kombu - handles retrying under tolerable/transient failures). + :param transport_options: transport specific options (see: + http://kombu.readthedocs.org/ for what these + options imply and are expected to be) + :param retry_options: retry specific options + (see: :py:attr:`~.proxy.Proxy.DEFAULT_RETRY_OPTIONS`) """ def __init__(self, exchange, topic, tasks, From f795df075b3687f200d8343009c4dd4497ecb5aa Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Tue, 27 Jan 2015 17:26:00 +0000 Subject: [PATCH 231/240] Updated from global requirements Change-Id: Id5908ac0ea4300f8b0eb660c107cfcf0452e6490 --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 96ab9440..293ec5dc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -24,7 +24,7 @@ kazoo>=1.3.1 # PyMySQL or MySQL-python depending on the python version the tests are being # ran in (MySQL-python is currently preferred for 2.x environments, since # it has been used in openstack for the longest). -alembic>=0.7.1 +alembic>=0.7.2 psycopg2 # Docs build jobs need these packages. From 80888c638cc4170ab83d168741743532e2bb5d9f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 27 Dec 2014 07:12:08 -0800 Subject: [PATCH 232/240] Use monotonic time when/if available Instead of using a time that can change depending on ntpd or other time adjustments use a monotonically increasing time if it's available from the underlying python library to avoid these types of time-shifts problems in the first place. Change-Id: Ib775a44026b9828536c905a1ed41f527358c0d39 --- taskflow/tests/unit/test_types.py | 29 +++--- .../tests/unit/worker_based/test_protocol.py | 12 +-- .../tests/unit/worker_based/test_types.py | 14 +-- taskflow/types/timing.py | 97 +++++++++++++++++-- taskflow/utils/misc.py | 29 ++++++ 5 files changed, 143 insertions(+), 38 deletions(-) diff --git a/taskflow/tests/unit/test_types.py b/taskflow/tests/unit/test_types.py index c13b263b..ebb8b672 100644 --- a/taskflow/tests/unit/test_types.py +++ b/taskflow/tests/unit/test_types.py @@ -15,7 +15,6 @@ # under the License. import networkx as nx -from oslo_utils import timeutils import six from taskflow import exceptions as excp @@ -123,8 +122,8 @@ class TreeTest(test.TestCase): class StopWatchTest(test.TestCase): def setUp(self): super(StopWatchTest, self).setUp() - timeutils.set_time_override() - self.addCleanup(timeutils.clear_time_override) + tt.StopWatch.set_now_override(now=0) + self.addCleanup(tt.StopWatch.clear_overrides) def test_no_states(self): watch = tt.StopWatch() @@ -137,23 +136,23 @@ class StopWatchTest(test.TestCase): def test_backwards(self): watch = tt.StopWatch(0.1) watch.start() - timeutils.advance_time_seconds(0.5) + tt.StopWatch.advance_time_seconds(0.5) self.assertTrue(watch.expired()) - timeutils.advance_time_seconds(-1.0) + tt.StopWatch.advance_time_seconds(-1.0) self.assertFalse(watch.expired()) self.assertEqual(0.0, watch.elapsed()) def test_expiry(self): watch = tt.StopWatch(0.1) watch.start() - timeutils.advance_time_seconds(0.2) + tt.StopWatch.advance_time_seconds(0.2) self.assertTrue(watch.expired()) def test_not_expired(self): watch = tt.StopWatch(0.1) watch.start() - timeutils.advance_time_seconds(0.05) + tt.StopWatch.advance_time_seconds(0.05) self.assertFalse(watch.expired()) def test_no_expiry(self): @@ -163,7 +162,7 @@ class StopWatchTest(test.TestCase): def test_elapsed(self): watch = tt.StopWatch() watch.start() - timeutils.advance_time_seconds(0.2) + tt.StopWatch.advance_time_seconds(0.2) # NOTE(harlowja): Allow for a slight variation by using 0.19. self.assertGreaterEqual(0.19, watch.elapsed()) @@ -180,17 +179,17 @@ class StopWatchTest(test.TestCase): def test_pause_resume(self): watch = tt.StopWatch() watch.start() - timeutils.advance_time_seconds(0.05) + tt.StopWatch.advance_time_seconds(0.05) watch.stop() elapsed = watch.elapsed() self.assertAlmostEqual(elapsed, watch.elapsed()) watch.resume() - timeutils.advance_time_seconds(0.05) + tt.StopWatch.advance_time_seconds(0.05) self.assertNotEqual(elapsed, watch.elapsed()) def test_context_manager(self): with tt.StopWatch() as watch: - timeutils.advance_time_seconds(0.05) + tt.StopWatch.advance_time_seconds(0.05) self.assertGreater(0.01, watch.elapsed()) def test_splits(self): @@ -203,7 +202,7 @@ class StopWatchTest(test.TestCase): self.assertEqual(watch.splits[0].elapsed, watch.splits[0].length) - timeutils.advance_time_seconds(0.05) + tt.StopWatch.advance_time_seconds(0.05) watch.split() splits = watch.splits self.assertEqual(2, len(splits)) @@ -221,16 +220,16 @@ class StopWatchTest(test.TestCase): watch = tt.StopWatch() watch.start() - timeutils.advance_time_seconds(1) + tt.StopWatch.advance_time_seconds(1) self.assertEqual(1, watch.elapsed()) - timeutils.advance_time_seconds(10) + tt.StopWatch.advance_time_seconds(10) self.assertEqual(11, watch.elapsed()) self.assertEqual(1, watch.elapsed(maximum=1)) watch.stop() self.assertEqual(11, watch.elapsed()) - timeutils.advance_time_seconds(10) + tt.StopWatch.advance_time_seconds(10) self.assertEqual(11, watch.elapsed()) self.assertEqual(0, watch.elapsed(maximum=-1)) diff --git a/taskflow/tests/unit/worker_based/test_protocol.py b/taskflow/tests/unit/worker_based/test_protocol.py index e5da38f2..5436df3c 100644 --- a/taskflow/tests/unit/worker_based/test_protocol.py +++ b/taskflow/tests/unit/worker_based/test_protocol.py @@ -15,7 +15,6 @@ # under the License. from concurrent import futures -from oslo_utils import timeutils from oslo_utils import uuidutils from taskflow.engines.action_engine import executor @@ -24,6 +23,7 @@ from taskflow import exceptions as excp from taskflow import test from taskflow.tests import utils from taskflow.types import failure +from taskflow.types import timing class TestProtocolValidation(test.TestCase): @@ -94,8 +94,8 @@ class TestProtocol(test.TestCase): def setUp(self): super(TestProtocol, self).setUp() - timeutils.set_time_override() - self.addCleanup(timeutils.clear_time_override) + timing.StopWatch.set_now_override() + self.addCleanup(timing.StopWatch.clear_overrides) self.task = utils.DummyTask() self.task_uuid = 'task-uuid' self.task_action = 'execute' @@ -166,19 +166,19 @@ class TestProtocol(test.TestCase): def test_pending_not_expired(self): req = self.request() - timeutils.advance_time_seconds(self.timeout - 1) + timing.StopWatch.set_offset_override(self.timeout - 1) self.assertFalse(req.expired) def test_pending_expired(self): req = self.request() - timeutils.advance_time_seconds(self.timeout + 1) + timing.StopWatch.set_offset_override(self.timeout + 1) self.assertTrue(req.expired) def test_running_not_expired(self): request = self.request() request.transition(pr.PENDING) request.transition(pr.RUNNING) - timeutils.advance_time_seconds(self.timeout + 1) + timing.StopWatch.set_offset_override(self.timeout + 1) self.assertFalse(request.expired) def test_set_result(self): diff --git a/taskflow/tests/unit/worker_based/test_types.py b/taskflow/tests/unit/worker_based/test_types.py index 6541575a..e1bf949b 100644 --- a/taskflow/tests/unit/worker_based/test_types.py +++ b/taskflow/tests/unit/worker_based/test_types.py @@ -14,12 +14,10 @@ # License for the specific language governing permissions and limitations # under the License. -import datetime import threading import time from oslo.utils import reflection -from oslo.utils import timeutils from taskflow.engines.worker_based import protocol as pr from taskflow.engines.worker_based import types as worker_types @@ -33,6 +31,7 @@ class TestWorkerTypes(test.TestCase): def setUp(self): super(TestWorkerTypes, self).setUp() + self.addCleanup(timing.StopWatch.clear_overrides) self.task = utils.DummyTask() self.task_uuid = 'task-uuid' self.task_action = 'execute' @@ -52,15 +51,12 @@ class TestWorkerTypes(test.TestCase): def test_requests_cache_expiry(self): # Mock out the calls the underlying objects will soon use to return # times that we can control more easily... - now = timeutils.utcnow() overrides = [ - now, - now, - now + datetime.timedelta(seconds=1), - now + datetime.timedelta(seconds=self.timeout + 1), + 0, + 1, + self.timeout + 1, ] - timeutils.set_time_override(overrides) - self.addCleanup(timeutils.clear_time_override) + timing.StopWatch.set_now_override(overrides) cache = worker_types.RequestsCache() cache[self.task_uuid] = self.request() diff --git a/taskflow/types/timing.py b/taskflow/types/timing.py index 8f868431..8e60e6b8 100644 --- a/taskflow/types/timing.py +++ b/taskflow/types/timing.py @@ -15,10 +15,14 @@ # under the License. from oslo_utils import reflection -from oslo_utils import timeutils +from taskflow.utils import misc from taskflow.utils import threading_utils +# Find a monotonic providing time (or fallback to using time.time() +# which isn't *always* accurate but will suffice). +_now = misc.find_monotonic(allow_time_time=True) + class Timeout(object): """An object which represents a timeout. @@ -86,6 +90,12 @@ class StopWatch(object): _STARTED = 'STARTED' _STOPPED = 'STOPPED' + """ + Class variables that should only be used for testing purposes only... + """ + _now_offset = None + _now_override = None + def __init__(self, duration=None): if duration is not None: if duration < 0: @@ -105,7 +115,7 @@ class StopWatch(object): """ if self._state == self._STARTED: return self - self._started_at = timeutils.utcnow() + self._started_at = self._now() self._stopped_at = None self._state = self._STARTED self._splits = [] @@ -121,7 +131,7 @@ class StopWatch(object): if self._state == self._STARTED: elapsed = self.elapsed() if self._splits: - length = max(0.0, elapsed - self._splits[-1].elapsed) + length = self._delta_seconds(self._splits[-1].elapsed, elapsed) else: length = elapsed self._splits.append(Split(elapsed, length)) @@ -137,17 +147,88 @@ class StopWatch(object): self.start() return self + @classmethod + def clear_overrides(cls): + """Clears all overrides/offsets. + + **Only to be used for testing (affects all watch instances).** + """ + cls._now_override = None + cls._now_offset = None + + @classmethod + def set_offset_override(cls, offset): + """Sets a offset that is applied to each time fetch. + + **Only to be used for testing (affects all watch instances).** + """ + cls._now_offset = offset + + @classmethod + def advance_time_seconds(cls, offset): + """Advances/sets a offset that is applied to each time fetch. + + NOTE(harlowja): if a previous offset exists (not ``None``) then this + offset will be added onto the existing one (if you want to reset + the offset completely use the :meth:`.set_offset_override` + method instead). + + **Only to be used for testing (affects all watch instances).** + """ + if cls._now_offset is None: + cls.set_offset_override(offset) + else: + cls.set_offset_override(cls._now_offset + offset) + + @classmethod + def set_now_override(cls, now=None): + """Sets time override to use (if none, then current time is fetched). + + NOTE(harlowja): if a list/tuple is provided then the first element of + the list will be used (and removed) each time a time fetch occurs (once + it becomes empty the override/s will no longer be applied). If a + numeric value is provided then it will be used (and never removed + until the override(s) are cleared via the :meth:`.clear_overrides` + method). + + **Only to be used for testing (affects all watch instances).** + """ + if isinstance(now, (list, tuple)): + cls._now_override = list(now) + else: + if now is None: + now = _now() + cls._now_override = now + + @staticmethod + def _delta_seconds(earlier, later): + return max(0.0, later - earlier) + + @classmethod + def _now(cls): + if cls._now_override is not None: + if isinstance(cls._now_override, list): + try: + now = cls._now_override.pop(0) + except IndexError: + now = _now() + else: + now = cls._now_override + else: + now = _now() + if cls._now_offset is not None: + now = now + cls._now_offset + return now + def elapsed(self, maximum=None): """Returns how many seconds have elapsed.""" if self._state not in (self._STOPPED, self._STARTED): raise RuntimeError("Can not get the elapsed time of a stopwatch" " if it has not been started/stopped") if self._state == self._STOPPED: - elapsed = max(0.0, float(timeutils.delta_seconds( - self._started_at, self._stopped_at))) + elapsed = self._delta_seconds(self._started_at, self._stopped_at) else: - elapsed = max(0.0, float(timeutils.delta_seconds( - self._started_at, timeutils.utcnow()))) + elapsed = self._delta_seconds(self._started_at, self._now()) if maximum is not None and elapsed > maximum: elapsed = max(0.0, maximum) return elapsed @@ -201,6 +282,6 @@ class StopWatch(object): if self._state != self._STARTED: raise RuntimeError("Can not stop a stopwatch that has not been" " started") - self._stopped_at = timeutils.utcnow() + self._stopped_at = self._now() self._state = self._STOPPED return self diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 8905906a..299082ae 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -23,6 +23,7 @@ import os import re import sys import threading +import time import types from oslo_serialization import jsonutils @@ -44,6 +45,34 @@ NUMERIC_TYPES = six.integer_types + (float,) # see RFC 3986 section 3.1 _SCHEME_REGEX = re.compile(r"^([A-Za-z][A-Za-z0-9+.-]*):") +_MONOTONIC_LOCATIONS = tuple([ + # The built-in/expected location in python3.3+ + 'time.monotonic', + # NOTE(harlowja): Try to use the pypi module that provides this + # functionality for older versions of python less than 3.3 so that + # they to can benefit from better timing... + # + # See: http://pypi.python.org/pypi/monotonic + 'monotonic.monotonic', +]) + + +def find_monotonic(allow_time_time=False): + """Tries to find a monotonic time providing function (and returns it).""" + for import_str in _MONOTONIC_LOCATIONS: + mod_str, _sep, attr_str = import_str.rpartition('.') + mod = importutils.try_import(mod_str) + if mod is None: + continue + func = getattr(mod, attr_str, None) + if func is not None: + return func + # Finally give up and use time.time (which isn't monotonic)... + if allow_time_time: + return time.time + else: + return None + def merge_uri(uri, conf): """Merges a parsed uri into the given configuration dictionary. From e9226ca96d98534a74dec45a46409f800f01a6fc Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 28 Jan 2015 08:50:28 -0800 Subject: [PATCH 233/240] Add docstring to wbe proxy to denote not for public use Change-Id: I9bd59ae5d58ef3ace960509903403252fbed6277 --- taskflow/engines/worker_based/proxy.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/taskflow/engines/worker_based/proxy.py b/taskflow/engines/worker_based/proxy.py index 1770d562..4d4fe257 100644 --- a/taskflow/engines/worker_based/proxy.py +++ b/taskflow/engines/worker_based/proxy.py @@ -42,7 +42,10 @@ _TransportDetails = collections.namedtuple('_TransportDetails', class Proxy(object): - """A proxy processes messages from/to the named exchange.""" + """A proxy processes messages from/to the named exchange. + + For **internal** usage only (not for public consumption). + """ # Settings that are by default used for consumers/producers to reconnect # under tolerable/transient failures... From 2e43b67c40d853a4df545dd685ce0942158f50b8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 28 Jan 2015 09:13:25 -0800 Subject: [PATCH 234/240] Add note about publicly consumable types Add a note to the docs that describes that these are made for public usage; and that even so they still may be moved to new supported libraries at various points in the future. Change-Id: I172531ee07878eb5ef46821a5f3dc2fc3ce8c439 --- doc/source/types.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/source/types.rst b/doc/source/types.rst index fb9580af..3a11bea2 100644 --- a/doc/source/types.rst +++ b/doc/source/types.rst @@ -2,6 +2,18 @@ Types ----- +.. note:: + + Even though these types **are** made for public consumption and usage + should be encouraged/easily possible it should be noted that these may be + moved out to new libraries at various points in the future (for example + the ``FSM`` code *may* move to its own oslo supported ``automaton`` library + at some point in the future [#f1]_). If you are using these + types **without** using the rest of this library it is **strongly** + encouraged that you be a vocal proponent of getting these made + into *isolated* libraries (as using these types in this manner is not + the expected and/or desired usage). + Cache ===== @@ -46,3 +58,6 @@ Tree ==== .. automodule:: taskflow.types.tree + +.. [#f1] See: https://review.openstack.org/#/c/141961 for a proposal to do + do this. From d5128cf51a554c2aa0f50e4b48b953933fc33f89 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 28 Jan 2015 15:55:39 -0800 Subject: [PATCH 235/240] Stopwatch usage cleanup/tweak Instead of optionally creating a stopwatch when a provided timeout is not none (to avoid the stopwatch leftover() method raising a error) just allow the stopwatch leftover() method to not raise when no duration is provided to avoid these repeated styles of usage/checks in the first place. By default the leftover() method still raises an error (a new keyword argument is now accepted to turn off this behavior). Change-Id: If934ee6e6855adbb6975cd6ea41e273d40e73dac --- taskflow/engines/action_engine/executor.py | 15 +++++++------- taskflow/engines/worker_based/types.py | 12 ++++------- taskflow/jobs/backends/impl_zookeeper.py | 12 ++++------- taskflow/tests/unit/test_types.py | 7 +++++++ taskflow/types/latch.py | 15 ++++++-------- taskflow/types/timing.py | 23 +++++++++++++++------- 6 files changed, 45 insertions(+), 39 deletions(-) diff --git a/taskflow/engines/action_engine/executor.py b/taskflow/engines/action_engine/executor.py index 9b794d41..b271beb8 100644 --- a/taskflow/engines/action_engine/executor.py +++ b/taskflow/engines/action_engine/executor.py @@ -175,10 +175,11 @@ class _WaitWorkItem(object): 'kind': _KIND_COMPLETE_ME, } if self._channel.put(message): - w = timing.StopWatch().start() + watch = timing.StopWatch() + watch.start() self._barrier.wait() LOG.blather("Waited %s seconds until task '%s' %s emitted" - " notifications were depleted", w.elapsed(), + " notifications were depleted", watch.elapsed(), self._task, sent_events) def __call__(self): @@ -303,11 +304,11 @@ class _Dispatcher(object): " %s to target '%s'", kind, sender, target) def run(self, queue): - w = timing.StopWatch(duration=self._dispatch_periodicity) + watch = timing.StopWatch(duration=self._dispatch_periodicity) while (not self._dead.is_set() or (self._stop_when_empty and self._targets)): - w.restart() - leftover = w.leftover() + watch.restart() + leftover = watch.leftover() while leftover: try: message = queue.get(timeout=leftover) @@ -315,8 +316,8 @@ class _Dispatcher(object): break else: self._dispatch(message) - leftover = w.leftover() - leftover = w.leftover() + leftover = watch.leftover() + leftover = watch.leftover() if leftover: self._dead.wait(leftover) diff --git a/taskflow/engines/worker_based/types.py b/taskflow/engines/worker_based/types.py index 3d8aa632..d8a7e413 100644 --- a/taskflow/engines/worker_based/types.py +++ b/taskflow/engines/worker_based/types.py @@ -157,17 +157,13 @@ class TopicWorkers(object): """ if workers <= 0: raise ValueError("Worker amount must be greater than zero") - w = None - if timeout is not None: - w = tt.StopWatch(timeout).start() + watch = tt.StopWatch(duration=timeout) + watch.start() with self._cond: while len(self._workers) < workers: - if w is not None and w.expired(): + if watch.expired(): return max(0, workers - len(self._workers)) - timeout = None - if w is not None: - timeout = w.leftover() - self._cond.wait(timeout) + self._cond.wait(watch.leftover(return_none=True)) return 0 def get_worker_for_task(self, task): diff --git a/taskflow/jobs/backends/impl_zookeeper.py b/taskflow/jobs/backends/impl_zookeeper.py index 1fe74500..3e52f65b 100644 --- a/taskflow/jobs/backends/impl_zookeeper.py +++ b/taskflow/jobs/backends/impl_zookeeper.py @@ -661,13 +661,12 @@ class ZookeeperJobBoard(base.NotifyingJobBoard): def wait(self, timeout=None): # Wait until timeout expires (or forever) for jobs to appear. - watch = None - if timeout is not None: - watch = tt.StopWatch(duration=float(timeout)).start() + watch = tt.StopWatch(duration=timeout) + watch.start() with self._job_cond: while True: if not self._known_jobs: - if watch is not None and watch.expired(): + if watch.expired(): raise excp.NotFound("Expired waiting for jobs to" " arrive; waited %s seconds" % watch.elapsed()) @@ -676,10 +675,7 @@ class ZookeeperJobBoard(base.NotifyingJobBoard): # when we acquire the condition that there will actually # be jobs (especially if we are spuriously awaken), so we # must recalculate the amount of time we really have left. - timeout = None - if watch is not None: - timeout = watch.leftover() - self._job_cond.wait(timeout) + self._job_cond.wait(watch.leftover(return_none=True)) else: it = ZookeeperJobBoardIterator(self) it._jobs.extend(self._fetch_jobs()) diff --git a/taskflow/tests/unit/test_types.py b/taskflow/tests/unit/test_types.py index ebb8b672..1e639767 100644 --- a/taskflow/tests/unit/test_types.py +++ b/taskflow/tests/unit/test_types.py @@ -125,6 +125,13 @@ class StopWatchTest(test.TestCase): tt.StopWatch.set_now_override(now=0) self.addCleanup(tt.StopWatch.clear_overrides) + def test_leftover_no_duration(self): + watch = tt.StopWatch() + watch.start() + self.assertRaises(RuntimeError, watch.leftover) + self.assertRaises(RuntimeError, watch.leftover, return_none=False) + self.assertIsNone(watch.leftover(return_none=True)) + def test_no_states(self): watch = tt.StopWatch() self.assertRaises(RuntimeError, watch.stop) diff --git a/taskflow/types/latch.py b/taskflow/types/latch.py index db6e56f3..3e279787 100644 --- a/taskflow/types/latch.py +++ b/taskflow/types/latch.py @@ -54,15 +54,12 @@ class Latch(object): timeout expires then this will return True, otherwise it will return False. """ - w = None - if timeout is not None: - w = tt.StopWatch(timeout).start() + watch = tt.StopWatch(duration=timeout) + watch.start() with self._cond: while self._count > 0: - if w is not None: - if w.expired(): - return False - else: - timeout = w.leftover() - self._cond.wait(timeout) + if watch.expired(): + return False + else: + self._cond.wait(watch.leftover(return_none=True)) return True diff --git a/taskflow/types/timing.py b/taskflow/types/timing.py index 8e60e6b8..da3938dc 100644 --- a/taskflow/types/timing.py +++ b/taskflow/types/timing.py @@ -245,23 +245,32 @@ class StopWatch(object): except RuntimeError: pass - def leftover(self): - """Returns how many seconds are left until the watch expires.""" - if self._duration is None: - raise RuntimeError("Can not get the leftover time of a watch that" - " has no duration") + def leftover(self, return_none=False): + """Returns how many seconds are left until the watch expires. + + :param return_none: when ``True`` instead of raising a ``RuntimeError`` + when no duration has been set this call will + return ``None`` instead. + :type return_none: boolean + """ if self._state != self._STARTED: raise RuntimeError("Can not get the leftover time of a stopwatch" " that has not been started") + if self._duration is None: + if not return_none: + raise RuntimeError("Can not get the leftover time of a watch" + " that has no duration") + else: + return None return max(0.0, self._duration - self.elapsed()) def expired(self): """Returns if the watch has expired (ie, duration provided elapsed).""" - if self._duration is None: - return False if self._state is None: raise RuntimeError("Can not check if a stopwatch has expired" " if it has not been started/stopped") + if self._duration is None: + return False if self.elapsed() > self._duration: return True return False From 344b3f6c4de80465f03c3f6a7148bbfe69f17ef8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 29 Jan 2015 00:58:22 -0800 Subject: [PATCH 236/240] Use kombu socket.timeout alias instead of socket.timeout Use the kombu timeout error alias to socket.timeout to be more resilient against kombu ever changing this exception type to something different. Change-Id: I53c01db67742c65ee49264fd0d8d36417cec3ec1 --- taskflow/engines/worker_based/proxy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskflow/engines/worker_based/proxy.py b/taskflow/engines/worker_based/proxy.py index f4046234..02639270 100644 --- a/taskflow/engines/worker_based/proxy.py +++ b/taskflow/engines/worker_based/proxy.py @@ -15,9 +15,9 @@ # under the License. import collections -import socket import kombu +from kombu import exceptions as kombu_exceptions import six from taskflow.engines.worker_based import dispatcher @@ -191,7 +191,7 @@ class Proxy(object): def _drain(conn, timeout): try: conn.drain_events(timeout=timeout) - except socket.timeout: + except kombu_exceptions.TimeoutError: pass def _drain_errback(exc, interval): From 43d70ebe773b7f40afba108e962d68f1fe1c6ff3 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 29 Jan 2015 15:11:41 -0800 Subject: [PATCH 237/240] Use the class defined constant instead of raw strings In examples and documentation it is better if we recommend good practices for using the notifications and the good/better practice would be to use the class defined constant instead of using a raw string (in-case taskflow ever changes that constant value sometime in the future). Change-Id: Ied8fc88747f8635de4aa776095e7e0195d6043aa --- doc/source/notifications.rst | 6 ++++-- taskflow/examples/build_a_car.py | 17 ++++++++++------- taskflow/examples/simple_linear_listening.py | 7 +++++-- taskflow/examples/wbe_event_sender.py | 4 +++- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/doc/source/notifications.rst b/doc/source/notifications.rst index 3fd35e2b..c0dbe4e2 100644 --- a/doc/source/notifications.rst +++ b/doc/source/notifications.rst @@ -7,6 +7,8 @@ Notifications and listeners from taskflow import task from taskflow.patterns import linear_flow from taskflow import engines + from taskflow.types import notifier + ANY = notifier.Notifier.ANY -------- Overview @@ -57,7 +59,7 @@ A basic example is: >>> flo = linear_flow.Flow("cat-dog").add( ... CatTalk(), DogTalk(provides="dog")) >>> eng = engines.load(flo, store={'meow': 'meow', 'woof': 'woof'}) - >>> eng.notifier.register("*", flow_transition) + >>> eng.notifier.register(ANY, flow_transition) >>> eng.run() Flow 'cat-dog' transition to state RUNNING meow @@ -93,7 +95,7 @@ A basic example is: >>> flo.add(CatTalk(), DogTalk(provides="dog")) >>> eng = engines.load(flo, store={'meow': 'meow', 'woof': 'woof'}) - >>> eng.task_notifier.register("*", task_transition) + >>> eng.task_notifier.register(ANY, task_transition) >>> eng.run() Task 'CatTalk' transition to state RUNNING meow diff --git a/taskflow/examples/build_a_car.py b/taskflow/examples/build_a_car.py index 1655f2a6..02be020e 100644 --- a/taskflow/examples/build_a_car.py +++ b/taskflow/examples/build_a_car.py @@ -31,6 +31,9 @@ import taskflow.engines from taskflow.patterns import graph_flow as gf from taskflow.patterns import linear_flow as lf from taskflow import task +from taskflow.types import notifier + +ANY = notifier.Notifier.ANY import example_utils as eu # noqa @@ -160,11 +163,11 @@ spec = { engine = taskflow.engines.load(flow, store={'spec': spec.copy()}) -# This registers all (*) state transitions to trigger a call to the flow_watch -# function for flow state transitions, and registers the same all (*) state -# transitions for task state transitions. -engine.notifier.register('*', flow_watch) -engine.task_notifier.register('*', task_watch) +# This registers all (ANY) state transitions to trigger a call to the +# flow_watch function for flow state transitions, and registers the +# same all (ANY) state transitions for task state transitions. +engine.notifier.register(ANY, flow_watch) +engine.task_notifier.register(ANY, task_watch) eu.print_wrapped("Building a car") engine.run() @@ -176,8 +179,8 @@ engine.run() spec['doors'] = 5 engine = taskflow.engines.load(flow, store={'spec': spec.copy()}) -engine.notifier.register('*', flow_watch) -engine.task_notifier.register('*', task_watch) +engine.notifier.register(ANY, flow_watch) +engine.task_notifier.register(ANY, task_watch) eu.print_wrapped("Building a wrong car that doesn't match specification") try: diff --git a/taskflow/examples/simple_linear_listening.py b/taskflow/examples/simple_linear_listening.py index 04f9f14e..d14c82c4 100644 --- a/taskflow/examples/simple_linear_listening.py +++ b/taskflow/examples/simple_linear_listening.py @@ -28,6 +28,9 @@ sys.path.insert(0, top_dir) import taskflow.engines from taskflow.patterns import linear_flow as lf from taskflow import task +from taskflow.types import notifier + +ANY = notifier.Notifier.ANY # INTRO: In this example we create two tasks (this time as functions instead # of task subclasses as in the simple_linear.py example), each of which ~calls~ @@ -92,8 +95,8 @@ engine = taskflow.engines.load(flow, store={ # notification objects that a engine exposes. The usage of a '*' (kleene star) # here means that we want to be notified on all state changes, if you want to # restrict to a specific state change, just register that instead. -engine.notifier.register('*', flow_watch) -engine.task_notifier.register('*', task_watch) +engine.notifier.register(ANY, flow_watch) +engine.task_notifier.register(ANY, task_watch) # And now run! engine.run() diff --git a/taskflow/examples/wbe_event_sender.py b/taskflow/examples/wbe_event_sender.py index 38b6bfd9..e5b075ac 100644 --- a/taskflow/examples/wbe_event_sender.py +++ b/taskflow/examples/wbe_event_sender.py @@ -34,6 +34,8 @@ from taskflow import task from taskflow.types import notifier from taskflow.utils import threading_utils +ANY = notifier.Notifier.ANY + # INTRO: This examples shows how to use a remote workers event notification # attribute to proxy back task event notifications to the controlling process. # @@ -94,7 +96,7 @@ WORKER_CONF = { def run(engine_options): reporter = EventReporter() - reporter.notifier.register(notifier.Notifier.ANY, event_receiver) + reporter.notifier.register(ANY, event_receiver) flow = lf.Flow('event-reporter').add(reporter) eng = engines.load(flow, engine='worker-based', **engine_options) eng.run() From df6fb033494c0851b2f01f9e9a7f19c8bbf398b1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 29 Jan 2015 00:59:56 -0800 Subject: [PATCH 238/240] Remove duplicated 'do' in types documentation Change-Id: I20dda6eb1dc0d34a8714efc5d9dd6a0328f195bc --- doc/source/types.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/types.rst b/doc/source/types.rst index 3a11bea2..001e15ae 100644 --- a/doc/source/types.rst +++ b/doc/source/types.rst @@ -59,5 +59,5 @@ Tree .. automodule:: taskflow.types.tree -.. [#f1] See: https://review.openstack.org/#/c/141961 for a proposal to do +.. [#f1] See: https://review.openstack.org/#/c/141961 for a proposal to do this. From 99b92aed5de1d0868266ce63dec4f3b0e4a636d1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 26 Jan 2015 13:56:34 -0800 Subject: [PATCH 239/240] Add and use a nicer kombu message formatter Since the kombu message object that is recieved has no useful __str__ or __repr__ and on reception and processing we should the message (and the delivery tag) it is nice to have a more useful formatting of that message for debugging and such (where more detail about the messages are quite useful to see). Change-Id: I6730b10122a5de1a0a074525931f312fbd97b8c0 --- doc/source/utils.rst | 5 ++ taskflow/engines/worker_based/dispatcher.py | 17 +++-- taskflow/engines/worker_based/executor.py | 10 ++- taskflow/engines/worker_based/proxy.py | 3 +- taskflow/engines/worker_based/server.py | 38 +++++----- .../tests/unit/worker_based/test_server.py | 22 +++--- taskflow/utils/kombu_utils.py | 73 +++++++++++++++++++ 7 files changed, 126 insertions(+), 42 deletions(-) create mode 100644 taskflow/utils/kombu_utils.py diff --git a/doc/source/utils.rst b/doc/source/utils.rst index d3c726f0..1f774663 100644 --- a/doc/source/utils.rst +++ b/doc/source/utils.rst @@ -28,6 +28,11 @@ Kazoo .. automodule:: taskflow.utils.kazoo_utils +Kombu +~~~~~ + +.. automodule:: taskflow.utils.kombu_utils + Locks ~~~~~ diff --git a/taskflow/engines/worker_based/dispatcher.py b/taskflow/engines/worker_based/dispatcher.py index 7ea8947c..385fd13d 100644 --- a/taskflow/engines/worker_based/dispatcher.py +++ b/taskflow/engines/worker_based/dispatcher.py @@ -19,6 +19,7 @@ import six from taskflow import exceptions as excp from taskflow import logging +from taskflow.utils import kombu_utils as ku LOG = logging.getLogger(__name__) @@ -70,7 +71,7 @@ class TypeDispatcher(object): LOG.critical("Couldn't requeue %r, reason:%r", message.delivery_tag, exc, exc_info=True) else: - LOG.debug("AMQP message %r requeued.", message.delivery_tag) + LOG.debug("Message '%s' was requeued.", ku.DelayedPretty(message)) def _process_message(self, data, message, message_type): handler = self._handlers.get(message_type) @@ -78,7 +79,7 @@ class TypeDispatcher(object): message.reject_log_error(logger=LOG, errors=(kombu_exc.MessageStateError,)) LOG.warning("Unexpected message type: '%s' in message" - " %r", message_type, message.delivery_tag) + " '%s'", message_type, ku.DelayedPretty(message)) else: if isinstance(handler, (tuple, list)): handler, validator = handler @@ -87,15 +88,15 @@ class TypeDispatcher(object): except excp.InvalidFormat as e: message.reject_log_error( logger=LOG, errors=(kombu_exc.MessageStateError,)) - LOG.warn("Message: %r, '%s' was rejected due to it being" + LOG.warn("Message '%s' (%s) was rejected due to it being" " in an invalid format: %s", - message.delivery_tag, message_type, e) + ku.DelayedPretty(message), message_type, e) return message.ack_log_error(logger=LOG, errors=(kombu_exc.MessageStateError,)) if message.acknowledged: - LOG.debug("AMQP message %r acknowledged.", - message.delivery_tag) + LOG.debug("Message '%s' was acknowledged.", + ku.DelayedPretty(message)) handler(data, message) else: message.reject_log_error(logger=LOG, @@ -103,7 +104,7 @@ class TypeDispatcher(object): def on_message(self, data, message): """This method is called on incoming messages.""" - LOG.debug("Got message: %r", message.delivery_tag) + LOG.debug("Received message '%s'", ku.DelayedPretty(message)) if self._collect_requeue_votes(data, message): self._requeue_log_error(message, errors=(kombu_exc.MessageStateError,)) @@ -114,6 +115,6 @@ class TypeDispatcher(object): message.reject_log_error( logger=LOG, errors=(kombu_exc.MessageStateError,)) LOG.warning("The 'type' message property is missing" - " in message %r", message.delivery_tag) + " in message '%s'", ku.DelayedPretty(message)) else: self._process_message(data, message, message_type) diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index 8290ba61..6ece1b84 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -26,6 +26,7 @@ from taskflow import exceptions as exc from taskflow import logging from taskflow import task as task_atom from taskflow.types import timing as tt +from taskflow.utils import kombu_utils as ku from taskflow.utils import misc from taskflow.utils import threading_utils as tu @@ -73,7 +74,8 @@ class WorkerTaskExecutor(executor.TaskExecutor): def _process_notify(self, notify, message): """Process notify message from remote side.""" LOG.debug("Started processing notify message '%s'", - message.delivery_tag) + ku.DelayedPretty(message)) + topic = notify['topic'] tasks = notify['tasks'] @@ -91,11 +93,13 @@ class WorkerTaskExecutor(executor.TaskExecutor): def _process_response(self, response, message): """Process response from remote side.""" LOG.debug("Started processing response message '%s'", - message.delivery_tag) + ku.DelayedPretty(message)) try: task_uuid = message.properties['correlation_id'] except KeyError: - LOG.warning("The 'correlation_id' message property is missing") + LOG.warning("The 'correlation_id' message property is" + " missing in message '%s'", + ku.DelayedPretty(message)) else: request = self._requests_cache.get(task_uuid) if request is not None: diff --git a/taskflow/engines/worker_based/proxy.py b/taskflow/engines/worker_based/proxy.py index 02639270..505ead2a 100644 --- a/taskflow/engines/worker_based/proxy.py +++ b/taskflow/engines/worker_based/proxy.py @@ -176,7 +176,8 @@ class Proxy(object): LOG.exception('Publishing error: %s', exc) LOG.info('Retry triggering in %s seconds', interval) - LOG.debug("Sending '%s' using routing keys %s", msg, routing_keys) + LOG.debug("Sending '%s' message using routing keys %s", + msg, routing_keys) with kombu.connections[self._conn].acquire(block=True) as conn: with conn.Producer() as producer: ensure_kwargs = self._ensure_options.copy() diff --git a/taskflow/engines/worker_based/server.py b/taskflow/engines/worker_based/server.py index 6e64f1cd..5bb0b2ab 100644 --- a/taskflow/engines/worker_based/server.py +++ b/taskflow/engines/worker_based/server.py @@ -23,6 +23,7 @@ from taskflow.engines.worker_based import proxy from taskflow import logging from taskflow.types import failure as ft from taskflow.types import notifier as nt +from taskflow.utils import kombu_utils as ku from taskflow.utils import misc LOG = logging.getLogger(__name__) @@ -142,13 +143,14 @@ class Server(object): def _process_notify(self, notify, message): """Process notify message and reply back.""" - LOG.debug("Started processing notify message %r", message.delivery_tag) + LOG.debug("Started processing notify message '%s'", + ku.DelayedPretty(message)) try: reply_to = message.properties['reply_to'] except KeyError: LOG.warn("The 'reply_to' message property is missing" - " in received notify message %r", message.delivery_tag, - exc_info=True) + " in received notify message '%s'", + ku.DelayedPretty(message), exc_info=True) else: response = pr.Notify(topic=self._topic, tasks=self._endpoints.keys()) @@ -156,13 +158,13 @@ class Server(object): self._proxy.publish(response, routing_key=reply_to) except Exception: LOG.critical("Failed to send reply to '%s' with notify" - " response %s", reply_to, response, + " response '%s'", reply_to, response, exc_info=True) def _process_request(self, request, message): """Process request message and reply back.""" - LOG.debug("Started processing request message %r", - message.delivery_tag) + LOG.debug("Started processing request message '%s'", + ku.DelayedPretty(message)) try: # NOTE(skudriashev): parse broker message first to get # the `reply_to` and the `task_uuid` parameters to have @@ -170,8 +172,8 @@ class Server(object): # in the first place...). reply_to, task_uuid = self._parse_message(message) except ValueError: - LOG.warn("Failed to parse request attributes from message %r", - message.delivery_tag, exc_info=True) + LOG.warn("Failed to parse request attributes from message '%s'", + ku.DelayedPretty(message), exc_info=True) return else: # prepare reply callback @@ -185,8 +187,8 @@ class Server(object): arguments['task_uuid'] = task_uuid except ValueError: with misc.capture_failure() as failure: - LOG.warn("Failed to parse request contents from message %r", - message.delivery_tag, exc_info=True) + LOG.warn("Failed to parse request contents from message '%s'", + ku.DelayedPretty(message), exc_info=True) reply_callback(result=failure.to_dict()) return @@ -196,8 +198,8 @@ class Server(object): except KeyError: with misc.capture_failure() as failure: LOG.warn("The '%s' task endpoint does not exist, unable" - " to continue processing request message %r", - task_cls, message.delivery_tag, exc_info=True) + " to continue processing request message '%s'", + task_cls, ku.DelayedPretty(message), exc_info=True) reply_callback(result=failure.to_dict()) return else: @@ -207,8 +209,8 @@ class Server(object): with misc.capture_failure() as failure: LOG.warn("The '%s' handler does not exist on task endpoint" " '%s', unable to continue processing request" - " message %r", action, endpoint, - message.delivery_tag, exc_info=True) + " message '%s'", action, endpoint, + ku.DelayedPretty(message), exc_info=True) reply_callback(result=failure.to_dict()) return else: @@ -217,8 +219,8 @@ class Server(object): except Exception: with misc.capture_failure() as failure: LOG.warn("The '%s' task '%s' generation for request" - " message %r failed", endpoint, action, - message.delivery_tag, exc_info=True) + " message '%s' failed", endpoint, action, + ku.DelayedPretty(message), exc_info=True) reply_callback(result=failure.to_dict()) return else: @@ -245,8 +247,8 @@ class Server(object): except Exception: with misc.capture_failure() as failure: LOG.warn("The '%s' endpoint '%s' execution for request" - " message %r failed", endpoint, action, - message.delivery_tag, exc_info=True) + " message '%s' failed", endpoint, action, + ku.DelayedPretty(message), exc_info=True) reply_callback(result=failure.to_dict()) else: if isinstance(result, ft.Failure): diff --git a/taskflow/tests/unit/worker_based/test_server.py b/taskflow/tests/unit/worker_based/test_server.py index 7a77e8df..1fb8aa5c 100644 --- a/taskflow/tests/unit/worker_based/test_server.py +++ b/taskflow/tests/unit/worker_based/test_server.py @@ -154,7 +154,7 @@ class TestServer(test.MockTestCase): s = self.server(reset_master_mock=True) s._reply(True, self.reply_to, self.task_uuid) - self.assertEqual(self.master_mock.mock_calls, [ + self.master_mock.assert_has_calls([ mock.call.Response(pr.FAILURE), mock.call.proxy.publish(self.response_inst_mock, self.reply_to, correlation_id=self.task_uuid) @@ -195,7 +195,7 @@ class TestServer(test.MockTestCase): mock.call.proxy.publish(self.response_inst_mock, self.reply_to, correlation_id=self.task_uuid) ] - self.assertEqual(master_mock_calls, self.master_mock.mock_calls) + self.master_mock.assert_has_calls(master_mock_calls) def test_process_request(self): # create server and process request @@ -211,7 +211,7 @@ class TestServer(test.MockTestCase): mock.call.proxy.publish(self.response_inst_mock, self.reply_to, correlation_id=self.task_uuid) ] - self.assertEqual(self.master_mock.mock_calls, master_mock_calls) + self.master_mock.assert_has_calls(master_mock_calls) @mock.patch("taskflow.engines.worker_based.server.LOG.warn") def test_process_request_parse_message_failure(self, mocked_exception): @@ -219,8 +219,6 @@ class TestServer(test.MockTestCase): request = self.make_request() s = self.server(reset_master_mock=True) s._process_request(request, self.message_mock) - - self.assertEqual(self.master_mock.mock_calls, []) self.assertTrue(mocked_exception.called) @mock.patch.object(failure.Failure, 'from_dict') @@ -245,7 +243,7 @@ class TestServer(test.MockTestCase): self.reply_to, correlation_id=self.task_uuid) ] - self.assertEqual(master_mock_calls, self.master_mock.mock_calls) + self.master_mock.assert_has_calls(master_mock_calls) @mock.patch.object(failure.Failure, 'to_dict') def test_process_request_endpoint_not_found(self, to_mock): @@ -266,7 +264,7 @@ class TestServer(test.MockTestCase): self.reply_to, correlation_id=self.task_uuid) ] - self.assertEqual(self.master_mock.mock_calls, master_mock_calls) + self.master_mock.assert_has_calls(master_mock_calls) @mock.patch.object(failure.Failure, 'to_dict') def test_process_request_execution_failure(self, to_mock): @@ -288,7 +286,7 @@ class TestServer(test.MockTestCase): self.reply_to, correlation_id=self.task_uuid) ] - self.assertEqual(self.master_mock.mock_calls, master_mock_calls) + self.master_mock.assert_has_calls(master_mock_calls) @mock.patch.object(failure.Failure, 'to_dict') def test_process_request_task_failure(self, to_mock): @@ -312,7 +310,7 @@ class TestServer(test.MockTestCase): self.reply_to, correlation_id=self.task_uuid) ] - self.assertEqual(self.master_mock.mock_calls, master_mock_calls) + self.master_mock.assert_has_calls(master_mock_calls) def test_start(self): self.server(reset_master_mock=True).start() @@ -321,7 +319,7 @@ class TestServer(test.MockTestCase): master_mock_calls = [ mock.call.proxy.start() ] - self.assertEqual(self.master_mock.mock_calls, master_mock_calls) + self.master_mock.assert_has_calls(master_mock_calls) def test_wait(self): server = self.server(reset_master_mock=True) @@ -333,7 +331,7 @@ class TestServer(test.MockTestCase): mock.call.proxy.start(), mock.call.proxy.wait() ] - self.assertEqual(self.master_mock.mock_calls, master_mock_calls) + self.master_mock.assert_has_calls(master_mock_calls) def test_stop(self): self.server(reset_master_mock=True).stop() @@ -342,4 +340,4 @@ class TestServer(test.MockTestCase): master_mock_calls = [ mock.call.proxy.stop() ] - self.assertEqual(self.master_mock.mock_calls, master_mock_calls) + self.master_mock.assert_has_calls(master_mock_calls) diff --git a/taskflow/utils/kombu_utils.py b/taskflow/utils/kombu_utils.py new file mode 100644 index 00000000..8ace067b --- /dev/null +++ b/taskflow/utils/kombu_utils.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Keys extracted from the message properties when formatting... +_MSG_PROPERTIES = tuple([ + 'correlation_id', + 'delivery_info/routing_key', + 'type', +]) + + +class DelayedPretty(object): + """Wraps a message and delays prettifying it until requested.""" + + def __init__(self, message): + self._message = message + self._message_pretty = None + + def __str__(self): + if self._message_pretty is None: + self._message_pretty = _prettify_message(self._message) + return self._message_pretty + + +def _get_deep(properties, *keys): + """Get a final key among a list of keys (each with its own sub-dict).""" + for key in keys: + properties = properties[key] + return properties + + +def _prettify_message(message): + """Kombu doesn't currently have a useful ``__str__()`` or ``__repr__()``. + + This provides something decent(ish) for debugging (or other purposes) so + that messages are more nice and understandable.... + + TODO(harlowja): submit something into kombu to fix/adjust this. + """ + if message.content_type is not None: + properties = { + 'content_type': message.content_type, + } + else: + properties = {} + for name in _MSG_PROPERTIES: + segments = name.split("/") + try: + value = _get_deep(message.properties, *segments) + except (KeyError, ValueError, TypeError): + pass + else: + if value is not None: + properties[segments[-1]] = value + if message.body is not None: + properties['body_length'] = len(message.body) + return "%(delivery_tag)s: %(properties)s" % { + 'delivery_tag': message.delivery_tag, + 'properties': properties, + } From 19f9674877902e7e67b6c48df04571b040fdef50 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 26 Jan 2015 19:48:26 -0800 Subject: [PATCH 240/240] Abstract out the worker finding from the WBE engine To be able to easily plug-in future types of ways to get which topics (and tasks) workers exist on (and can perform) and to identify and keep this information up-to date refactor the functionality that currently does this using periodic messages into a finder type and a periodic function that exists on it (that will be periodically activated by an updated and improved periodic worker). Part of blueprint wbe-worker-info Change-Id: Ib3ae29758af3d244b4ac4624ac380caf88b159fd --- doc/source/types.rst | 5 + taskflow/engines/worker_based/dispatcher.py | 40 ++-- taskflow/engines/worker_based/executor.py | 59 +++--- taskflow/engines/worker_based/proxy.py | 19 +- taskflow/engines/worker_based/server.py | 3 +- taskflow/engines/worker_based/types.py | 185 ++++++++++-------- taskflow/tests/unit/test_types.py | 133 +++++++++++++ .../unit/worker_based/test_dispatcher.py | 12 +- .../tests/unit/worker_based/test_executor.py | 19 +- .../tests/unit/worker_based/test_server.py | 4 +- .../tests/unit/worker_based/test_types.py | 72 +++---- taskflow/types/periodic.py | 179 +++++++++++++++++ 12 files changed, 529 insertions(+), 201 deletions(-) create mode 100644 taskflow/types/periodic.py diff --git a/doc/source/types.rst b/doc/source/types.rst index 001e15ae..47ba7e48 100644 --- a/doc/source/types.rst +++ b/doc/source/types.rst @@ -44,6 +44,11 @@ Notifier .. automodule:: taskflow.types.notifier +Periodic +======== + +.. automodule:: taskflow.types.periodic + Table ===== diff --git a/taskflow/engines/worker_based/dispatcher.py b/taskflow/engines/worker_based/dispatcher.py index 385fd13d..13470e08 100644 --- a/taskflow/engines/worker_based/dispatcher.py +++ b/taskflow/engines/worker_based/dispatcher.py @@ -15,7 +15,6 @@ # under the License. from kombu import exceptions as kombu_exc -import six from taskflow import exceptions as excp from taskflow import logging @@ -27,14 +26,35 @@ LOG = logging.getLogger(__name__) class TypeDispatcher(object): """Receives messages and dispatches to type specific handlers.""" - def __init__(self, type_handlers): - self._handlers = dict(type_handlers) - self._requeue_filters = [] + def __init__(self, type_handlers=None, requeue_filters=None): + if type_handlers is not None: + self._type_handlers = dict(type_handlers) + else: + self._type_handlers = {} + if requeue_filters is not None: + self._requeue_filters = list(requeue_filters) + else: + self._requeue_filters = [] - def add_requeue_filter(self, callback): - """Add a callback that can *request* message requeuing. + @property + def type_handlers(self): + """Dictionary of message type -> callback to handle that message. - The callback will be activated before the message has been acked and + The callback(s) will be activated by looking for a message + property 'type' and locating a callback in this dictionary that maps + to that type; if one is found it is expected to be a callback that + accepts two positional parameters; the first being the message data + and the second being the message object. If a callback is not found + then the message is rejected and it will be up to the underlying + message transport to determine what this means/implies... + """ + return self._type_handlers + + @property + def requeue_filters(self): + """List of filters (callbacks) to request a message to be requeued. + + The callback(s) will be activated before the message has been acked and it can be used to instruct the dispatcher to requeue the message instead of processing it. The callback, when called, will be provided two positional parameters; the first being the message data and the @@ -42,9 +62,7 @@ class TypeDispatcher(object): filter should return a truthy object if the message should be requeued and a falsey object if it should not. """ - if not six.callable(callback): - raise ValueError("Requeue filter callback must be callable") - self._requeue_filters.append(callback) + return self._requeue_filters def _collect_requeue_votes(self, data, message): # Returns how many of the filters asked for the message to be requeued. @@ -74,7 +92,7 @@ class TypeDispatcher(object): LOG.debug("Message '%s' was requeued.", ku.DelayedPretty(message)) def _process_message(self, data, message, message_type): - handler = self._handlers.get(message_type) + handler = self._type_handlers.get(message_type) if handler is None: message.reject_log_error(logger=LOG, errors=(kombu_exc.MessageStateError,)) diff --git a/taskflow/engines/worker_based/executor.py b/taskflow/engines/worker_based/executor.py index 6ece1b84..a55229f1 100644 --- a/taskflow/engines/worker_based/executor.py +++ b/taskflow/engines/worker_based/executor.py @@ -25,7 +25,7 @@ from taskflow.engines.worker_based import types as wt from taskflow import exceptions as exc from taskflow import logging from taskflow import task as task_atom -from taskflow.types import timing as tt +from taskflow.types import periodic from taskflow.utils import kombu_utils as ku from taskflow.utils import misc from taskflow.utils import threading_utils as tu @@ -41,51 +41,41 @@ class WorkerTaskExecutor(executor.TaskExecutor): url=None, transport=None, transport_options=None, retry_options=None): self._uuid = uuid - self._topics = topics self._requests_cache = wt.RequestsCache() - self._workers = wt.TopicWorkers() self._transition_timeout = transition_timeout type_handlers = { - pr.NOTIFY: [ - self._process_notify, - functools.partial(pr.Notify.validate, response=True), - ], pr.RESPONSE: [ self._process_response, pr.Response.validate, ], } - self._proxy = proxy.Proxy(uuid, exchange, type_handlers, + self._proxy = proxy.Proxy(uuid, exchange, + type_handlers=type_handlers, on_wait=self._on_wait, url=url, transport=transport, transport_options=transport_options, retry_options=retry_options) - self._periodic = wt.PeriodicWorker(tt.Timeout(pr.NOTIFY_PERIOD), - [self._notify_topics]) + # NOTE(harlowja): This is the most simplest finder impl. that + # doesn't have external dependencies (outside of what this engine + # already requires); it though does create periodic 'polling' traffic + # to workers to 'learn' of the tasks they can perform (and requires + # pre-existing knowledge of the topics those workers are on to gather + # and update this information). + self._finder = wt.ProxyWorkerFinder(uuid, self._proxy, topics) + self._finder.on_worker = self._on_worker self._helpers = tu.ThreadBundle() self._helpers.bind(lambda: tu.daemon_thread(self._proxy.start), after_start=lambda t: self._proxy.wait(), before_join=lambda t: self._proxy.stop()) - self._helpers.bind(lambda: tu.daemon_thread(self._periodic.start), - before_join=lambda t: self._periodic.stop(), - after_join=lambda t: self._periodic.reset(), - before_start=lambda t: self._periodic.reset()) + p_worker = periodic.PeriodicWorker.create([self._finder]) + if p_worker: + self._helpers.bind(lambda: tu.daemon_thread(p_worker.start), + before_join=lambda t: p_worker.stop(), + after_join=lambda t: p_worker.reset(), + before_start=lambda t: p_worker.reset()) - def _process_notify(self, notify, message): - """Process notify message from remote side.""" - LOG.debug("Started processing notify message '%s'", - ku.DelayedPretty(message)) - - topic = notify['topic'] - tasks = notify['tasks'] - - # Add worker info to the cache - worker = self._workers.add(topic, tasks) - LOG.debug("Received notification about worker '%s' (%s" - " total workers are currently known)", worker, - len(self._workers)) - - # Publish waiting requests + def _on_worker(self, worker): + """Process new worker that has arrived (and fire off any work).""" for request in self._requests_cache.get_waiting_requests(worker): if request.transition_and_log_error(pr.PENDING, logger=LOG): self._publish_request(request, worker) @@ -174,7 +164,7 @@ class WorkerTaskExecutor(executor.TaskExecutor): request.result.add_done_callback(lambda fut: cleaner()) # Get task's worker and publish request if worker was found. - worker = self._workers.get_worker_for_task(task) + worker = self._finder.get_worker_for_task(task) if worker is not None: # NOTE(skudriashev): Make sure request is set to the PENDING state # before putting it into the requests cache to prevent the notify @@ -208,10 +198,6 @@ class WorkerTaskExecutor(executor.TaskExecutor): del self._requests_cache[request.uuid] request.set_result(failure) - def _notify_topics(self): - """Cyclically called to publish notify message to each topic.""" - self._proxy.publish(pr.Notify(), self._topics, reply_to=self._uuid) - def execute_task(self, task, task_uuid, arguments, progress_callback=None): return self._submit_task(task, task_uuid, pr.EXECUTE, arguments, @@ -232,7 +218,8 @@ class WorkerTaskExecutor(executor.TaskExecutor): return how many workers are still needed, otherwise it will return zero. """ - return self._workers.wait_for_workers(workers=workers, timeout=timeout) + return self._finder.wait_for_workers(workers=workers, + timeout=timeout) def start(self): """Starts proxy thread and associated topic notification thread.""" @@ -242,4 +229,4 @@ class WorkerTaskExecutor(executor.TaskExecutor): """Stops proxy thread and associated topic notification thread.""" self._helpers.stop() self._requests_cache.clear(self._handle_expired_request) - self._workers.clear() + self._finder.clear() diff --git a/taskflow/engines/worker_based/proxy.py b/taskflow/engines/worker_based/proxy.py index 505ead2a..e9d2ec22 100644 --- a/taskflow/engines/worker_based/proxy.py +++ b/taskflow/engines/worker_based/proxy.py @@ -68,19 +68,19 @@ class Proxy(object): # value is valid... _RETRY_INT_OPTS = frozenset(['max_retries']) - def __init__(self, topic, exchange, type_handlers, - on_wait=None, url=None, + def __init__(self, topic, exchange, + type_handlers=None, on_wait=None, url=None, transport=None, transport_options=None, retry_options=None): self._topic = topic self._exchange_name = exchange self._on_wait = on_wait self._running = threading_utils.Event() - self._dispatcher = dispatcher.TypeDispatcher(type_handlers) - self._dispatcher.add_requeue_filter( + self._dispatcher = dispatcher.TypeDispatcher( # NOTE(skudriashev): Process all incoming messages only if proxy is # running, otherwise requeue them. - lambda data, message: not self.is_running) + requeue_filters=[lambda data, message: not self.is_running], + type_handlers=type_handlers) ensure_options = self.DEFAULT_RETRY_OPTIONS.copy() if retry_options is not None: @@ -112,11 +112,16 @@ class Proxy(object): # create exchange self._exchange = kombu.Exchange(name=self._exchange_name, - durable=False, - auto_delete=True) + durable=False, auto_delete=True) + + @property + def dispatcher(self): + """Dispatcher internally used to dispatch message(s) that match.""" + return self._dispatcher @property def connection_details(self): + """Details about the connection (read-only).""" # The kombu drivers seem to use 'N/A' when they don't have a version... driver_version = self._conn.transport.driver_version() if driver_version and driver_version.lower() == 'n/a': diff --git a/taskflow/engines/worker_based/server.py b/taskflow/engines/worker_based/server.py index 5bb0b2ab..949b4691 100644 --- a/taskflow/engines/worker_based/server.py +++ b/taskflow/engines/worker_based/server.py @@ -59,7 +59,8 @@ class Server(object): pr.Request.validate, ], } - self._proxy = proxy.Proxy(topic, exchange, type_handlers, + self._proxy = proxy.Proxy(topic, exchange, + type_handlers=type_handlers, url=url, transport=transport, transport_options=transport_options, retry_options=retry_options) diff --git a/taskflow/engines/worker_based/types.py b/taskflow/engines/worker_based/types.py index d8a7e413..70185d52 100644 --- a/taskflow/engines/worker_based/types.py +++ b/taskflow/engines/worker_based/types.py @@ -14,17 +14,21 @@ # License for the specific language governing permissions and limitations # under the License. +import abc +import functools import itertools -import logging import random import threading -from oslo.utils import reflection +from oslo_utils import reflection import six from taskflow.engines.worker_based import protocol as pr +from taskflow import logging from taskflow.types import cache as base +from taskflow.types import periodic from taskflow.types import timing as tt +from taskflow.utils import kombu_utils as ku LOG = logging.getLogger(__name__) @@ -91,8 +95,37 @@ class TopicWorker(object): return r -class TopicWorkers(object): - """A collection of topic based workers.""" +@six.add_metaclass(abc.ABCMeta) +class WorkerFinder(object): + """Base class for worker finders...""" + + def __init__(self): + self._cond = threading.Condition() + self.on_worker = None + + @abc.abstractmethod + def _total_workers(self): + """Returns how many workers are known.""" + + def wait_for_workers(self, workers=1, timeout=None): + """Waits for geq workers to notify they are ready to do work. + + NOTE(harlowja): if a timeout is provided this function will wait + until that timeout expires, if the amount of workers does not reach + the desired amount of workers before the timeout expires then this will + return how many workers are still needed, otherwise it will + return zero. + """ + if workers <= 0: + raise ValueError("Worker amount must be greater than zero") + watch = tt.StopWatch(duration=timeout) + watch.start() + with self._cond: + while self._total_workers() < workers: + if watch.expired(): + return max(0, workers - self._total_workers()) + self._cond.wait(watch.leftover(return_none=True)) + return 0 @staticmethod def _match_worker(task, available_workers): @@ -110,14 +143,30 @@ class TopicWorkers(object): else: return random.choice(available_workers) - def __init__(self): - self._workers = {} - self._cond = threading.Condition() - # Used to name workers with more useful identities... - self._counter = itertools.count() + @abc.abstractmethod + def get_worker_for_task(self, task): + """Gets a worker that can perform a given task.""" - def __len__(self): - return len(self._workers) + def clear(self): + pass + + +class ProxyWorkerFinder(WorkerFinder): + """Requests and receives responses about workers topic+task details.""" + + def __init__(self, uuid, proxy, topics): + super(ProxyWorkerFinder, self).__init__() + self._proxy = proxy + self._topics = topics + self._workers = {} + self._uuid = uuid + self._proxy.dispatcher.type_handlers.update({ + pr.NOTIFY: [ + self._process_response, + functools.partial(pr.Notify.validate, response=True), + ], + }) + self._counter = itertools.count() def _next_worker(self, topic, tasks, temporary=False): if not temporary: @@ -126,48 +175,54 @@ class TopicWorkers(object): else: return TopicWorker(topic, tasks) - def add(self, topic, tasks): + @periodic.periodic(pr.NOTIFY_PERIOD) + def beat(self): + """Cyclically called to publish notify message to each topic.""" + self._proxy.publish(pr.Notify(), self._topics, reply_to=self._uuid) + + def _total_workers(self): + return len(self._workers) + + def _add(self, topic, tasks): """Adds/updates a worker for the topic for the given tasks.""" + try: + worker = self._workers[topic] + # Check if we already have an equivalent worker, if so just + # return it... + if worker == self._next_worker(topic, tasks, temporary=True): + return (worker, False) + # This *fall through* is done so that if someone is using an + # active worker object that already exists that we just create + # a new one; so that the existing object doesn't get + # affected (workers objects are supposed to be immutable). + except KeyError: + pass + worker = self._next_worker(topic, tasks) + self._workers[topic] = worker + return (worker, True) + + def _process_response(self, response, message): + """Process notify message from remote side.""" + LOG.debug("Started processing notify message '%s'", + ku.DelayedPretty(message)) + topic = response['topic'] + tasks = response['tasks'] with self._cond: - try: - worker = self._workers[topic] - # Check if we already have an equivalent worker, if so just - # return it... - if worker == self._next_worker(topic, tasks, temporary=True): - return worker - # This *fall through* is done so that if someone is using an - # active worker object that already exists that we just create - # a new one; so that the existing object doesn't get - # affected (workers objects are supposed to be immutable). - except KeyError: - pass - worker = self._next_worker(topic, tasks) - self._workers[topic] = worker + worker, new_or_updated = self._add(topic, tasks) + if new_or_updated: + LOG.debug("Received notification about worker '%s' (%s" + " total workers are currently known)", worker, + self._total_workers()) + self._cond.notify_all() + if self.on_worker is not None and new_or_updated: + self.on_worker(worker) + + def clear(self): + with self._cond: + self._workers.clear() self._cond.notify_all() - return worker - - def wait_for_workers(self, workers=1, timeout=None): - """Waits for geq workers to notify they are ready to do work. - - NOTE(harlowja): if a timeout is provided this function will wait - until that timeout expires, if the amount of workers does not reach - the desired amount of workers before the timeout expires then this will - return how many workers are still needed, otherwise it will - return zero. - """ - if workers <= 0: - raise ValueError("Worker amount must be greater than zero") - watch = tt.StopWatch(duration=timeout) - watch.start() - with self._cond: - while len(self._workers) < workers: - if watch.expired(): - return max(0, workers - len(self._workers)) - self._cond.wait(watch.leftover(return_none=True)) - return 0 def get_worker_for_task(self, task): - """Gets a worker that can perform a given task.""" available_workers = [] with self._cond: for worker in six.itervalues(self._workers): @@ -177,37 +232,3 @@ class TopicWorkers(object): return self._match_worker(task, available_workers) else: return None - - def clear(self): - with self._cond: - self._workers.clear() - self._cond.notify_all() - - -class PeriodicWorker(object): - """Calls a set of functions when activated periodically. - - NOTE(harlowja): the provided timeout object determines the periodicity. - """ - def __init__(self, timeout, functors): - self._timeout = timeout - self._functors = [] - for f in functors: - self._functors.append((f, reflection.get_callable_name(f))) - - def start(self): - while not self._timeout.is_stopped(): - for (f, f_name) in self._functors: - LOG.debug("Calling periodic function '%s'", f_name) - try: - f() - except Exception: - LOG.warn("Failed to call periodic function '%s'", f_name, - exc_info=True) - self._timeout.wait() - - def stop(self): - self._timeout.interrupt() - - def reset(self): - self._timeout.reset() diff --git a/taskflow/tests/unit/test_types.py b/taskflow/tests/unit/test_types.py index 1e639767..4daea4bf 100644 --- a/taskflow/tests/unit/test_types.py +++ b/taskflow/tests/unit/test_types.py @@ -14,6 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. +import time + import networkx as nx import six @@ -21,9 +23,31 @@ from taskflow import exceptions as excp from taskflow import test from taskflow.types import fsm from taskflow.types import graph +from taskflow.types import latch +from taskflow.types import periodic from taskflow.types import table from taskflow.types import timing as tt from taskflow.types import tree +from taskflow.utils import threading_utils as tu + + +class PeriodicThingy(object): + def __init__(self): + self.capture = [] + + @periodic.periodic(0.01) + def a(self): + self.capture.append('a') + + @periodic.periodic(0.02) + def b(self): + self.capture.append('b') + + def c(self): + pass + + def d(self): + pass class GraphTest(test.TestCase): @@ -451,3 +475,112 @@ class FSMTest(test.TestCase): m.add_state('broken') self.assertRaises(ValueError, m.add_state, 'b', on_enter=2) self.assertRaises(ValueError, m.add_state, 'b', on_exit=2) + + +class PeriodicTest(test.TestCase): + + def test_invalid_periodic(self): + + def no_op(): + pass + + self.assertRaises(ValueError, periodic.periodic, -1) + + def test_valid_periodic(self): + + @periodic.periodic(2) + def no_op(): + pass + + self.assertTrue(getattr(no_op, '_periodic')) + self.assertEqual(2, getattr(no_op, '_periodic_spacing')) + self.assertEqual(True, getattr(no_op, '_periodic_run_immediately')) + + def test_scanning_periodic(self): + p = PeriodicThingy() + w = periodic.PeriodicWorker.create([p]) + self.assertEqual(2, len(w)) + + t = tu.daemon_thread(target=w.start) + t.start() + time.sleep(0.1) + w.stop() + t.join() + + b_calls = [c for c in p.capture if c == 'b'] + self.assertGreater(0, len(b_calls)) + a_calls = [c for c in p.capture if c == 'a'] + self.assertGreater(0, len(a_calls)) + + def test_periodic_single(self): + barrier = latch.Latch(5) + capture = [] + tombstone = tu.Event() + + @periodic.periodic(0.01) + def callee(): + barrier.countdown() + if barrier.needed == 0: + tombstone.set() + capture.append(1) + + w = periodic.PeriodicWorker([callee], tombstone=tombstone) + t = tu.daemon_thread(target=w.start) + t.start() + t.join() + + self.assertEqual(0, barrier.needed) + self.assertEqual(5, sum(capture)) + self.assertTrue(tombstone.is_set()) + + def test_immediate(self): + capture = [] + + @periodic.periodic(120, run_immediately=True) + def a(): + capture.append('a') + + w = periodic.PeriodicWorker([a]) + t = tu.daemon_thread(target=w.start) + t.start() + time.sleep(0.1) + w.stop() + t.join() + + a_calls = [c for c in capture if c == 'a'] + self.assertGreater(0, len(a_calls)) + + def test_period_double_no_immediate(self): + capture = [] + + @periodic.periodic(0.01, run_immediately=False) + def a(): + capture.append('a') + + @periodic.periodic(0.02, run_immediately=False) + def b(): + capture.append('b') + + w = periodic.PeriodicWorker([a, b]) + t = tu.daemon_thread(target=w.start) + t.start() + time.sleep(0.1) + w.stop() + t.join() + + b_calls = [c for c in capture if c == 'b'] + self.assertGreater(0, len(b_calls)) + a_calls = [c for c in capture if c == 'a'] + self.assertGreater(0, len(a_calls)) + + def test_start_nothing_error(self): + w = periodic.PeriodicWorker([]) + self.assertRaises(RuntimeError, w.start) + + def test_missing_function_attrs(self): + + def fake_periodic(): + pass + + cb = fake_periodic + self.assertRaises(ValueError, periodic.PeriodicWorker, [cb]) diff --git a/taskflow/tests/unit/worker_based/test_dispatcher.py b/taskflow/tests/unit/worker_based/test_dispatcher.py index db0a719c..21fccdcc 100644 --- a/taskflow/tests/unit/worker_based/test_dispatcher.py +++ b/taskflow/tests/unit/worker_based/test_dispatcher.py @@ -41,12 +41,12 @@ class TestDispatcher(test.TestCase): def test_creation(self): on_hello = mock.MagicMock() handlers = {'hello': on_hello} - dispatcher.TypeDispatcher(handlers) + dispatcher.TypeDispatcher(type_handlers=handlers) def test_on_message(self): on_hello = mock.MagicMock() handlers = {'hello': on_hello} - d = dispatcher.TypeDispatcher(handlers) + d = dispatcher.TypeDispatcher(type_handlers=handlers) msg = mock_acked_message(properties={'type': 'hello'}) d.on_message("", msg) self.assertTrue(on_hello.called) @@ -54,15 +54,15 @@ class TestDispatcher(test.TestCase): self.assertTrue(msg.acknowledged) def test_on_rejected_message(self): - d = dispatcher.TypeDispatcher({}) + d = dispatcher.TypeDispatcher() msg = mock_acked_message(properties={'type': 'hello'}) d.on_message("", msg) self.assertTrue(msg.reject_log_error.called) self.assertFalse(msg.acknowledged) def test_on_requeue_message(self): - d = dispatcher.TypeDispatcher({}) - d.add_requeue_filter(lambda data, message: True) + d = dispatcher.TypeDispatcher() + d.requeue_filters.append(lambda data, message: True) msg = mock_acked_message() d.on_message("", msg) self.assertTrue(msg.requeue.called) @@ -71,7 +71,7 @@ class TestDispatcher(test.TestCase): def test_failed_ack(self): on_hello = mock.MagicMock() handlers = {'hello': on_hello} - d = dispatcher.TypeDispatcher(handlers) + d = dispatcher.TypeDispatcher(type_handlers=handlers) msg = mock_acked_message(ack_ok=False, properties={'type': 'hello'}) d.on_message("", msg) diff --git a/taskflow/tests/unit/worker_based/test_executor.py b/taskflow/tests/unit/worker_based/test_executor.py index 101031c4..e7831783 100644 --- a/taskflow/tests/unit/worker_based/test_executor.py +++ b/taskflow/tests/unit/worker_based/test_executor.py @@ -86,11 +86,12 @@ class TestWorkerTaskExecutor(test.MockTestCase): ex = self.executor(reset_master_mock=False) master_mock_calls = [ mock.call.Proxy(self.executor_uuid, self.executor_exchange, - mock.ANY, on_wait=ex._on_wait, + on_wait=ex._on_wait, url=self.broker_url, transport=mock.ANY, transport_options=mock.ANY, - retry_options=mock.ANY - ) + retry_options=mock.ANY, + type_handlers=mock.ANY), + mock.call.proxy.dispatcher.type_handlers.update(mock.ANY), ] self.assertEqual(self.master_mock.mock_calls, master_mock_calls) @@ -212,10 +213,8 @@ class TestWorkerTaskExecutor(test.MockTestCase): self.assertEqual(len(ex._requests_cache), 0) def test_execute_task(self): - self.message_mock.properties['type'] = pr.NOTIFY - notify = pr.Notify(topic=self.executor_topic, tasks=[self.task.name]) ex = self.executor() - ex._process_notify(notify.to_dict(), self.message_mock) + ex._finder._add(self.executor_topic, [self.task.name]) ex.execute_task(self.task, self.task_uuid, self.task_args) expected_calls = [ @@ -231,10 +230,8 @@ class TestWorkerTaskExecutor(test.MockTestCase): self.assertEqual(expected_calls, self.master_mock.mock_calls) def test_revert_task(self): - self.message_mock.properties['type'] = pr.NOTIFY - notify = pr.Notify(topic=self.executor_topic, tasks=[self.task.name]) ex = self.executor() - ex._process_notify(notify.to_dict(), self.message_mock) + ex._finder._add(self.executor_topic, [self.task.name]) ex.revert_task(self.task, self.task_uuid, self.task_args, self.task_result, self.task_failures) @@ -263,11 +260,9 @@ class TestWorkerTaskExecutor(test.MockTestCase): self.assertEqual(self.master_mock.mock_calls, expected_calls) def test_execute_task_publish_error(self): - self.message_mock.properties['type'] = pr.NOTIFY self.proxy_inst_mock.publish.side_effect = Exception('Woot!') - notify = pr.Notify(topic=self.executor_topic, tasks=[self.task.name]) ex = self.executor() - ex._process_notify(notify.to_dict(), self.message_mock) + ex._finder._add(self.executor_topic, [self.task.name]) ex.execute_task(self.task, self.task_uuid, self.task_args) expected_calls = [ diff --git a/taskflow/tests/unit/worker_based/test_server.py b/taskflow/tests/unit/worker_based/test_server.py index 1fb8aa5c..fea5d1cc 100644 --- a/taskflow/tests/unit/worker_based/test_server.py +++ b/taskflow/tests/unit/worker_based/test_server.py @@ -86,7 +86,7 @@ class TestServer(test.MockTestCase): # check calls master_mock_calls = [ mock.call.Proxy(self.server_topic, self.server_exchange, - mock.ANY, url=self.broker_url, + type_handlers=mock.ANY, url=self.broker_url, transport=mock.ANY, transport_options=mock.ANY, retry_options=mock.ANY) ] @@ -99,7 +99,7 @@ class TestServer(test.MockTestCase): # check calls master_mock_calls = [ mock.call.Proxy(self.server_topic, self.server_exchange, - mock.ANY, url=self.broker_url, + type_handlers=mock.ANY, url=self.broker_url, transport=mock.ANY, transport_options=mock.ANY, retry_options=mock.ANY) ] diff --git a/taskflow/tests/unit/worker_based/test_types.py b/taskflow/tests/unit/worker_based/test_types.py index e1bf949b..287283cf 100644 --- a/taskflow/tests/unit/worker_based/test_types.py +++ b/taskflow/tests/unit/worker_based/test_types.py @@ -14,23 +14,20 @@ # License for the specific language governing permissions and limitations # under the License. -import threading -import time - from oslo.utils import reflection from taskflow.engines.worker_based import protocol as pr from taskflow.engines.worker_based import types as worker_types from taskflow import test +from taskflow.test import mock from taskflow.tests import utils -from taskflow.types import latch from taskflow.types import timing -class TestWorkerTypes(test.TestCase): +class TestRequestCache(test.TestCase): def setUp(self): - super(TestWorkerTypes, self).setUp() + super(TestRequestCache, self).setUp() self.addCleanup(timing.StopWatch.clear_overrides) self.task = utils.DummyTask() self.task_uuid = 'task-uuid' @@ -76,6 +73,8 @@ class TestWorkerTypes(test.TestCase): self.assertEqual(1, len(matches)) self.assertEqual(2, len(cache)) + +class TestTopicWorker(test.TestCase): def test_topic_worker(self): worker = worker_types.TopicWorker("dummy-topic", [utils.DummyTask], identity="dummy") @@ -84,52 +83,37 @@ class TestWorkerTypes(test.TestCase): self.assertEqual('dummy', worker.identity) self.assertEqual('dummy-topic', worker.topic) - def test_single_topic_workers(self): - workers = worker_types.TopicWorkers() - w = workers.add('dummy-topic', [utils.DummyTask]) + +class TestProxyFinder(test.TestCase): + def test_single_topic_worker(self): + finder = worker_types.ProxyWorkerFinder('me', mock.MagicMock(), []) + w, emit = finder._add('dummy-topic', [utils.DummyTask]) self.assertIsNotNone(w) - self.assertEqual(1, len(workers)) - w2 = workers.get_worker_for_task(utils.DummyTask) + self.assertTrue(emit) + self.assertEqual(1, finder._total_workers()) + w2 = finder.get_worker_for_task(utils.DummyTask) self.assertEqual(w.identity, w2.identity) def test_multi_same_topic_workers(self): - workers = worker_types.TopicWorkers() - w = workers.add('dummy-topic', [utils.DummyTask]) + finder = worker_types.ProxyWorkerFinder('me', mock.MagicMock(), []) + w, emit = finder._add('dummy-topic', [utils.DummyTask]) self.assertIsNotNone(w) - w2 = workers.add('dummy-topic-2', [utils.DummyTask]) + self.assertTrue(emit) + w2, emit = finder._add('dummy-topic-2', [utils.DummyTask]) self.assertIsNotNone(w2) - w3 = workers.get_worker_for_task( + self.assertTrue(emit) + w3 = finder.get_worker_for_task( reflection.get_class_name(utils.DummyTask)) self.assertIn(w3.identity, [w.identity, w2.identity]) def test_multi_different_topic_workers(self): - workers = worker_types.TopicWorkers() + finder = worker_types.ProxyWorkerFinder('me', mock.MagicMock(), []) added = [] - added.append(workers.add('dummy-topic', [utils.DummyTask])) - added.append(workers.add('dummy-topic-2', [utils.DummyTask])) - added.append(workers.add('dummy-topic-3', [utils.NastyTask])) - self.assertEqual(3, len(workers)) - w = workers.get_worker_for_task(utils.NastyTask) - self.assertEqual(added[-1].identity, w.identity) - w = workers.get_worker_for_task(utils.DummyTask) - self.assertIn(w.identity, [w_a.identity for w_a in added[0:2]]) - - def test_periodic_worker(self): - barrier = latch.Latch(5) - to = timing.Timeout(0.01) - called_at = [] - - def callee(): - barrier.countdown() - if barrier.needed == 0: - to.interrupt() - called_at.append(time.time()) - - w = worker_types.PeriodicWorker(to, [callee]) - t = threading.Thread(target=w.start) - t.start() - t.join() - - self.assertEqual(0, barrier.needed) - self.assertEqual(5, len(called_at)) - self.assertTrue(to.is_stopped()) + added.append(finder._add('dummy-topic', [utils.DummyTask])) + added.append(finder._add('dummy-topic-2', [utils.DummyTask])) + added.append(finder._add('dummy-topic-3', [utils.NastyTask])) + self.assertEqual(3, finder._total_workers()) + w = finder.get_worker_for_task(utils.NastyTask) + self.assertEqual(added[-1][0].identity, w.identity) + w = finder.get_worker_for_task(utils.DummyTask) + self.assertIn(w.identity, [w_a[0].identity for w_a in added[0:2]]) diff --git a/taskflow/types/periodic.py b/taskflow/types/periodic.py new file mode 100644 index 00000000..bbb494d3 --- /dev/null +++ b/taskflow/types/periodic.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import heapq +import inspect + +from oslo_utils import reflection +import six + +from taskflow import logging +from taskflow.utils import misc +from taskflow.utils import threading_utils as tu + +LOG = logging.getLogger(__name__) + +# Find a monotonic providing time (or fallback to using time.time() +# which isn't *always* accurate but will suffice). +_now = misc.find_monotonic(allow_time_time=True) + +# Attributes expected on periodic tagged/decorated functions or methods... +_PERIODIC_ATTRS = tuple([ + '_periodic', + '_periodic_spacing', + '_periodic_run_immediately', +]) + + +def periodic(spacing, run_immediately=True): + """Tags a method/function as wanting/able to execute periodically.""" + + if spacing <= 0: + raise ValueError("Periodicity/spacing must be greater than" + " zero instead of %s" % spacing) + + def wrapper(f): + f._periodic = True + f._periodic_spacing = spacing + f._periodic_run_immediately = run_immediately + + @six.wraps(f) + def decorator(*args, **kwargs): + return f(*args, **kwargs) + + return decorator + + return wrapper + + +class PeriodicWorker(object): + """Calls a collection of callables periodically (sleeping as needed...). + + NOTE(harlowja): typically the :py:meth:`.start` method is executed in a + background thread so that the periodic callables are executed in + the background/asynchronously (using the defined periods to determine + when each is called). + """ + + @classmethod + def create(cls, objects, exclude_hidden=True): + """Automatically creates a worker by analyzing object(s) methods. + + Only picks up methods that have been tagged/decorated with + the :py:func:`.periodic` decorator (does not match against private + or protected methods unless explicitly requested to). + """ + callables = [] + for obj in objects: + for (name, member) in inspect.getmembers(obj): + if name.startswith("_") and exclude_hidden: + continue + if reflection.is_bound_method(member): + consume = True + for attr_name in _PERIODIC_ATTRS: + if not hasattr(member, attr_name): + consume = False + break + if consume: + callables.append(member) + return cls(callables) + + def __init__(self, callables, tombstone=None): + if tombstone is None: + self._tombstone = tu.Event() + else: + # Allows someone to share an event (if they so want to...) + self._tombstone = tombstone + almost_callables = list(callables) + for cb in almost_callables: + if not six.callable(cb): + raise ValueError("Periodic callback must be callable") + for attr_name in _PERIODIC_ATTRS: + if not hasattr(cb, attr_name): + raise ValueError("Periodic callback missing required" + " attribute '%s'" % attr_name) + self._callables = tuple((cb, reflection.get_callable_name(cb)) + for cb in almost_callables) + self._schedule = [] + self._immediates = [] + now = _now() + for i, (cb, cb_name) in enumerate(self._callables): + spacing = getattr(cb, '_periodic_spacing') + next_run = now + spacing + heapq.heappush(self._schedule, (next_run, i)) + for (cb, cb_name) in reversed(self._callables): + if getattr(cb, '_periodic_run_immediately', False): + self._immediates.append((cb, cb_name)) + + def __len__(self): + return len(self._callables) + + @staticmethod + def _safe_call(cb, cb_name, kind='periodic'): + try: + cb() + except Exception: + LOG.warn("Failed to call %s callable '%s'", + kind, cb_name, exc_info=True) + + def start(self): + """Starts running (will not stop/return until the tombstone is set). + + NOTE(harlowja): If this worker has no contained callables this raises + a runtime error and does not run since it is impossible to periodically + run nothing. + """ + if not self._callables: + raise RuntimeError("A periodic worker can not start" + " without any callables") + while not self._tombstone.is_set(): + if self._immediates: + cb, cb_name = self._immediates.pop() + LOG.debug("Calling immediate callable '%s'", cb_name) + self._safe_call(cb, cb_name, kind='immediate') + else: + # Figure out when we should run next (by selecting the + # minimum item from the heap, where the minimum should be + # the callable that needs to run next and has the lowest + # next desired run time). + now = _now() + next_run, i = heapq.heappop(self._schedule) + when_next = next_run - now + if when_next <= 0: + cb, cb_name = self._callables[i] + spacing = getattr(cb, '_periodic_spacing') + LOG.debug("Calling periodic callable '%s' (it runs every" + " %s seconds)", cb_name, spacing) + self._safe_call(cb, cb_name) + # Run again someday... + next_run = now + spacing + heapq.heappush(self._schedule, (next_run, i)) + else: + # Gotta wait... + heapq.heappush(self._schedule, (next_run, i)) + self._tombstone.wait(when_next) + + def stop(self): + """Sets the tombstone (this stops any further executions).""" + self._tombstone.set() + + def reset(self): + """Resets the tombstone and re-queues up any immediate executions.""" + self._tombstone.clear() + self._immediates = [] + for (cb, cb_name) in reversed(self._callables): + if getattr(cb, '_periodic_run_immediately', False): + self._immediates.append((cb, cb_name))