openstack/taskflow
Code Issues Proposed changes

Add ruff

Browse Source 
Change-Id: I2518e0cf928210acf9cfb2e5f4c19f973df64485
Signed-off-by: Stephen Finucane <stephenfin@redhat.com>
This commit is contained in:
Stephen Finucane
2026-03-11 10:43:22 +00:00
parent e2cabccf61
commit 2a6420a3ed
195 changed files with 9216 additions and 6116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
.pre-commit-config.yaml
View File

@@ -12,14 +12,15 @@ repos:
- id: debug-statements
- id: check-yaml
files: .*\.(yaml|yml)$
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.15.5
hooks:
- id: ruff-check
args: ['--fix', '--unsafe-fixes']
- id: ruff-format
- repo: https://opendev.org/openstack/hacking
rev: 8.0.0
hooks:
- id: hacking
additional_dependencies: []
exclude: '^(doc|releasenotes|tools)/.*$'
- repo: https://github.com/asottile/pyupgrade
rev: v3.21.2
hooks:
- id: pyupgrade
args: [--py310-plus]

2
doc/source/conf.py
View File

@@ -25,7 +25,7 @@ extensions = [
'sphinx.ext.extlinks',
'sphinx.ext.inheritance_diagram',
'sphinx.ext.viewcode',
'openstackdocstheme'
'openstackdocstheme',
]
# openstackdocstheme options

22
pyproject.toml
View File

@@ -95,7 +95,21 @@ parallel = "taskflow.engines.action_engine.engine:ParallelActionEngine"
worker-based = "taskflow.engines.worker_based.engine:WorkerBasedActionEngine"
workers = "taskflow.engines.worker_based.engine:WorkerBasedActionEngine"
[tool.setuptools]
packages = [
"taskflow"
]
[tool.setuptools.packages.find]
include = ["taskflow"]
[tool.ruff]
line-length = 79
[tool.ruff.format]
quote-style = "preserve"
docstring-code-format = true
[tool.ruff.lint]
select = ["E4", "E5", "E7", "E9", "F", "G", "LOG", "S"]
external = ["H"]
ignore = ["E402", "E721", "E731", "E741"]
[tool.ruff.lint.per-file-ignores]
"taskflow/examples/*" = ["S"]
"taskflow/tests/*" = ["S"]

10
releasenotes/source/conf.py
View File

@@ -88,9 +88,13 @@ html_static_path = ['_static']
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
('index', 'taskflowReleaseNotes.tex',
'taskflow Release Notes Documentation',
'taskflow Developers', 'manual'),
(
'index',
'taskflowReleaseNotes.tex',
'taskflow Release Notes Documentation',
'taskflow Developers',
'manual',
),
]

4
setup.py
View File

@@ -15,6 +15,4 @@
import setuptools
setuptools.setup(
setup_requires=['pbr>=2.0.0'],
pbr=True)
setuptools.setup(setup_requires=['pbr>=2.0.0'], pbr=True)

92
taskflow/atom.py
View File

@@ -52,8 +52,9 @@ def _save_as_to_mapping(save_as):
# NOTE(harlowja): this means that your atom will return a indexable
# object, like a list or tuple and the results can be mapped by index
# to that tuple/list that is returned for others to use.
return collections.OrderedDict((key, num)
for num, key in enumerate(save_as))
return collections.OrderedDict(
(key, num) for num, key in enumerate(save_as)
)
elif isinstance(save_as, _set_types):
# NOTE(harlowja): in the case where a set is given we will not be
# able to determine the numeric ordering in a reliable way (since it
@@ -61,8 +62,10 @@ def _save_as_to_mapping(save_as):
# result of the atom will be via the key itself.
return collections.OrderedDict((key, key) for key in save_as)
else:
raise TypeError('Atom provides parameter '
'should be str, set or tuple/list, not %r' % save_as)
raise TypeError(
'Atom provides parameter '
'should be str, set or tuple/list, not %r' % save_as
)
def _build_rebind_dict(req_args, rebind_args):
@@ -84,17 +87,19 @@ def _build_rebind_dict(req_args, rebind_args):
# Extra things were rebound, that may be because of *args
# or **kwargs (or some other reason); so just keep all of them
# using 1:1 rebinding...
rebind.update((a, a) for a in rebind_args[len(req_args):])
rebind.update((a, a) for a in rebind_args[len(req_args) :])
return rebind
elif isinstance(rebind_args, dict):
return rebind_args
else:
raise TypeError("Invalid rebind value '%s' (%s)"
% (rebind_args, type(rebind_args)))
raise TypeError(
"Invalid rebind value '%s' (%s)" % (rebind_args, type(rebind_args))
)
def _build_arg_mapping(atom_name, reqs, rebind_args, function, do_infer,
ignore_list=None):
def _build_arg_mapping(
atom_name, reqs, rebind_args, function, do_infer, ignore_list=None
):
"""Builds an input argument mapping for a given function.
Given a function, its requirements and a rebind mapping this helper
@@ -135,8 +140,9 @@ def _build_arg_mapping(atom_name, reqs, rebind_args, function, do_infer,
# Determine if there are optional arguments that we may or may not take.
if do_infer:
opt_args = sets.OrderedSet(all_args)
opt_args = opt_args - set(itertools.chain(required.keys(),
iter(ignore_list)))
opt_args = opt_args - set(
itertools.chain(required.keys(), iter(ignore_list))
)
optional = collections.OrderedDict((a, a) for a in opt_args)
else:
optional = collections.OrderedDict()
@@ -146,14 +152,17 @@ def _build_arg_mapping(atom_name, reqs, rebind_args, function, do_infer,
extra_args = sets.OrderedSet(required.keys())
extra_args -= all_args
if extra_args:
raise ValueError('Extra arguments given to atom %s: %s'
% (atom_name, list(extra_args)))
raise ValueError(
'Extra arguments given to atom %s: %s'
% (atom_name, list(extra_args))
)
# NOTE(imelnikov): don't use set to preserve order in error message
missing_args = [arg for arg in req_args if arg not in required]
if missing_args:
raise ValueError('Missing arguments for atom %s: %s'
% (atom_name, missing_args))
raise ValueError(
'Missing arguments for atom %s: %s' % (atom_name, missing_args)
)
return required, optional
@@ -244,9 +253,18 @@ class Atom(metaclass=abc.ABCMeta):
default_provides = None
def __init__(self, name=None, provides=None, requires=None,
auto_extract=True, rebind=None, inject=None,
ignore_list=None, revert_rebind=None, revert_requires=None):
def __init__(
self,
name=None,
provides=None,
requires=None,
auto_extract=True,
rebind=None,
inject=None,
ignore_list=None,
revert_rebind=None,
revert_requires=None,
):
if provides is None:
provides = self.default_provides
@@ -263,8 +281,9 @@ class Atom(metaclass=abc.ABCMeta):
self.rebind, exec_requires, self.optional = self._build_arg_mapping(
self.execute,
requires=requires,
rebind=rebind, auto_extract=auto_extract,
ignore_list=ignore_list
rebind=rebind,
auto_extract=auto_extract,
ignore_list=ignore_list,
)
revert_ignore = ignore_list + list(_default_revert_args)
@@ -273,10 +292,11 @@ class Atom(metaclass=abc.ABCMeta):
requires=revert_requires or requires,
rebind=revert_rebind or rebind,
auto_extract=auto_extract,
ignore_list=revert_ignore
ignore_list=revert_ignore,
)
(self.revert_rebind, addl_requires, self.revert_optional) = (
revert_mapping
)
(self.revert_rebind, addl_requires,
self.revert_optional) = revert_mapping
# TODO(bnemec): This should be documented as an ivar, but can't be due
# to https://github.com/sphinx-doc/sphinx/issues/2549
@@ -284,18 +304,30 @@ class Atom(metaclass=abc.ABCMeta):
#: requires to function.
self.requires = exec_requires.union(addl_requires)
def _build_arg_mapping(self, executor, requires=None, rebind=None,
auto_extract=True, ignore_list=None):
def _build_arg_mapping(
self,
executor,
requires=None,
rebind=None,
auto_extract=True,
ignore_list=None,
):
required, optional = _build_arg_mapping(self.name, requires, rebind,
executor, auto_extract,
ignore_list=ignore_list)
required, optional = _build_arg_mapping(
self.name,
requires,
rebind,
executor,
auto_extract,
ignore_list=ignore_list,
)
# Form the real rebind mapping, if a key name is the same as the
# key value, then well there is no rebinding happening, otherwise
# there will be.
rebind = collections.OrderedDict()
for (arg_name, bound_name) in itertools.chain(required.items(),
optional.items()):
for arg_name, bound_name in itertools.chain(
required.items(), optional.items()
):
rebind.setdefault(arg_name, bound_name)
requires = sets.OrderedSet(required.values())
optional = sets.OrderedSet(optional.values())

6
taskflow/conductors/backends/__init__.py
View File

@@ -34,10 +34,12 @@ def fetch(kind, name, jobboard, namespace=CONDUCTOR_NAMESPACE, **kwargs):
LOG.debug('Looking for %r conductor driver in %r', kind, namespace)
try:
mgr = stevedore.driver.DriverManager(
namespace, kind,
namespace,
kind,
invoke_on_load=True,
invoke_args=(name, jobboard),
invoke_kwds=kwargs)
invoke_kwds=kwargs,
)
return mgr.driver
except RuntimeError as e:
raise exc.NotFound("Could not find conductor %s" % (kind), e)

27
taskflow/conductors/backends/impl_blocking.py
View File

@@ -27,13 +27,24 @@ class BlockingConductor(impl_executor.ExecutorConductor):
def _executor_factory():
return futurist.SynchronousExecutor()
def __init__(self, name, jobboard,
persistence=None, engine=None,
engine_options=None, wait_timeout=None,
log=None, max_simultaneous_jobs=MAX_SIMULTANEOUS_JOBS):
def __init__(
self,
name,
jobboard,
persistence=None,
engine=None,
engine_options=None,
wait_timeout=None,
log=None,
max_simultaneous_jobs=MAX_SIMULTANEOUS_JOBS,
):
super().__init__(
name, jobboard,
persistence=persistence, engine=engine,
name,
jobboard,
persistence=persistence,
engine=engine,
engine_options=engine_options,
wait_timeout=wait_timeout, log=log,
max_simultaneous_jobs=max_simultaneous_jobs)
wait_timeout=wait_timeout,
log=log,
max_simultaneous_jobs=max_simultaneous_jobs,
)

197
taskflow/conductors/backends/impl_executor.py
View File

@@ -78,47 +78,72 @@ class ExecutorConductor(base.Conductor, metaclass=abc.ABCMeta):
"""
#: Exceptions that will **not** cause consumption to occur.
NO_CONSUME_EXCEPTIONS = tuple([
excp.ExecutionFailure,
excp.StorageFailure,
])
NO_CONSUME_EXCEPTIONS = tuple(
[
excp.ExecutionFailure,
excp.StorageFailure,
]
)
_event_factory = threading.Event
"""This attribute *can* be overridden by subclasses (for example if
an eventlet *green* event works better for the conductor user)."""
EVENTS_EMITTED = tuple([
'compilation_start', 'compilation_end',
'preparation_start', 'preparation_end',
'validation_start', 'validation_end',
'running_start', 'running_end',
'job_consumed', 'job_abandoned',
])
EVENTS_EMITTED = tuple(
[
'compilation_start',
'compilation_end',
'preparation_start',
'preparation_end',
'validation_start',
'validation_end',
'running_start',
'running_end',
'job_consumed',
'job_abandoned',
]
)
"""Events will be emitted for each of the events above. The event is
emitted to listeners registered with the conductor.
"""
def __init__(self, name, jobboard,
persistence=None, engine=None,
engine_options=None, wait_timeout=None,
log=None, max_simultaneous_jobs=MAX_SIMULTANEOUS_JOBS):
def __init__(
self,
name,
jobboard,
persistence=None,
engine=None,
engine_options=None,
wait_timeout=None,
log=None,
max_simultaneous_jobs=MAX_SIMULTANEOUS_JOBS,
):
super().__init__(
name, jobboard, persistence=persistence,
engine=engine, engine_options=engine_options)
name,
jobboard,
persistence=persistence,
engine=engine,
engine_options=engine_options,
)
self._wait_timeout = tt.convert_to_timeout(
value=wait_timeout, default_value=self.WAIT_TIMEOUT,
event_factory=self._event_factory)
value=wait_timeout,
default_value=self.WAIT_TIMEOUT,
event_factory=self._event_factory,
)
self._dead = self._event_factory()
self._log = misc.pick_first_not_none(log, self.LOG, LOG)
self._max_simultaneous_jobs = int(
misc.pick_first_not_none(max_simultaneous_jobs,
self.MAX_SIMULTANEOUS_JOBS))
misc.pick_first_not_none(
max_simultaneous_jobs, self.MAX_SIMULTANEOUS_JOBS
)
)
self._dispatched = set()
def _executor_factory(self):
"""Creates an executor to be used during dispatching."""
raise excp.NotImplementedError("This method must be implemented but"
" it has not been")
raise excp.NotImplementedError(
"This method must be implemented but it has not been"
)
def stop(self):
self._wait_timeout.interrupt()
@@ -134,8 +159,9 @@ class ExecutorConductor(base.Conductor, metaclass=abc.ABCMeta):
def _listeners_from_job(self, job, engine):
listeners = super()._listeners_from_job(job, engine)
listeners.append(logging_listener.LoggingListener(engine,
log=self._log))
listeners.append(
logging_listener.LoggingListener(engine, log=self._log)
)
return listeners
def _dispatch_job(self, job):
@@ -156,17 +182,22 @@ class ExecutorConductor(base.Conductor, metaclass=abc.ABCMeta):
has_suspended = False
for _state in engine.run_iter():
if not has_suspended and self._wait_timeout.is_stopped():
self._log.info("Conductor stopped, requesting "
"suspension of engine running "
"job %s", job)
self._log.info(
"Conductor stopped, requesting "
"suspension of engine running "
"job %s",
job,
)
engine.suspend()
has_suspended = True
try:
for stage_func, event_name in [(engine.compile, 'compilation'),
(engine.prepare, 'preparation'),
(engine.validate, 'validation'),
(_run_engine, 'running')]:
for stage_func, event_name in [
(engine.compile, 'compilation'),
(engine.prepare, 'preparation'),
(engine.validate, 'validation'),
(_run_engine, 'running'),
]:
self._notifier.notify("%s_start" % event_name, details)
stage_func()
self._notifier.notify("%s_end" % event_name, details)
@@ -177,23 +208,35 @@ class ExecutorConductor(base.Conductor, metaclass=abc.ABCMeta):
if consume:
self._log.warning(
"Job execution failed (consumption being"
" skipped): %s [%s failures]", job, len(e))
" skipped): %s [%s failures]",
job,
len(e),
)
else:
self._log.warning(
"Job execution failed (consumption"
" proceeding): %s [%s failures]", job, len(e))
" proceeding): %s [%s failures]",
job,
len(e),
)
# Show the failure/s + traceback (if possible)...
for i, f in enumerate(e):
self._log.warning("%s. %s", i + 1,
f.pformat(traceback=True))
self._log.warning(
"%s. %s", i + 1, f.pformat(traceback=True)
)
except self.NO_CONSUME_EXCEPTIONS:
self._log.warning("Job execution failed (consumption being"
" skipped): %s", job, exc_info=True)
self._log.warning(
"Job execution failed (consumption being skipped): %s",
job,
exc_info=True,
)
consume = False
except Exception:
self._log.warning(
"Job execution failed (consumption proceeding): %s",
job, exc_info=True)
job,
exc_info=True,
)
else:
if engine.storage.get_flow_state() == states.SUSPENDED:
self._log.info("Job execution was suspended: %s", job)
@@ -206,32 +249,43 @@ class ExecutorConductor(base.Conductor, metaclass=abc.ABCMeta):
try:
if consume:
self._jobboard.consume(job, self._name)
self._notifier.notify("job_consumed", {
'job': job,
'conductor': self,
'persistence': self._persistence,
})
self._notifier.notify(
"job_consumed",
{
'job': job,
'conductor': self,
'persistence': self._persistence,
},
)
elif trash:
self._jobboard.trash(job, self._name)
self._notifier.notify("job_trashed", {
'job': job,
'conductor': self,
'persistence': self._persistence,
})
self._notifier.notify(
"job_trashed",
{
'job': job,
'conductor': self,
'persistence': self._persistence,
},
)
else:
self._jobboard.abandon(job, self._name)
self._notifier.notify("job_abandoned", {
'job': job,
'conductor': self,
'persistence': self._persistence,
})
self._notifier.notify(
"job_abandoned",
{
'job': job,
'conductor': self,
'persistence': self._persistence,
},
)
except (excp.JobFailure, excp.NotFound):
if consume:
self._log.warn("Failed job consumption: %s", job,
exc_info=True)
self._log.warn(
"Failed job consumption: %s", job, exc_info=True
)
else:
self._log.warn("Failed job abandonment: %s", job,
exc_info=True)
self._log.warn(
"Failed job abandonment: %s", job, exc_info=True
)
def _on_job_done(self, job, fut):
consume = False
@@ -273,7 +327,8 @@ class ExecutorConductor(base.Conductor, metaclass=abc.ABCMeta):
if max_dispatches == 0:
raise StopIteration
fresh_period = timeutils.StopWatch(
duration=self.REFRESH_PERIODICITY)
duration=self.REFRESH_PERIODICITY
)
fresh_period.start()
while not is_stopped():
any_dispatched = False
@@ -284,28 +339,32 @@ class ExecutorConductor(base.Conductor, metaclass=abc.ABCMeta):
ensure_fresh = False
job_it = itertools.takewhile(
self._can_claim_more_jobs,
self._jobboard.iterjobs(ensure_fresh=ensure_fresh))
self._jobboard.iterjobs(ensure_fresh=ensure_fresh),
)
for job in job_it:
self._log.debug("Trying to claim job: %s", job)
try:
self._jobboard.claim(job, self._name)
except (excp.UnclaimableJob, excp.NotFound):
self._log.debug("Job already claimed or"
" consumed: %s", job)
self._log.debug(
"Job already claimed or consumed: %s", job
)
else:
try:
fut = executor.submit(self._dispatch_job, job)
except RuntimeError:
with excutils.save_and_reraise_exception():
self._log.warn("Job dispatch submitting"
" failed: %s", job)
self._log.warn(
"Job dispatch submitting failed: %s", job
)
self._try_finish_job(job, False)
else:
fut.job = job
self._dispatched.add(fut)
any_dispatched = True
fut.add_done_callback(
functools.partial(self._on_job_done, job))
functools.partial(self._on_job_done, job)
)
total_dispatched = next(dispatch_gen)
if not any_dispatched and not is_stopped():
self._wait_timeout.wait()
@@ -314,8 +373,9 @@ class ExecutorConductor(base.Conductor, metaclass=abc.ABCMeta):
# max dispatch number (which implies we should do no more work).
with excutils.save_and_reraise_exception():
if max_dispatches >= 0 and total_dispatched >= max_dispatches:
self._log.info("Maximum dispatch limit of %s reached",
max_dispatches)
self._log.info(
"Maximum dispatch limit of %s reached", max_dispatches
)
def run(self, max_dispatches=None):
self._dead.clear()
@@ -323,8 +383,7 @@ class ExecutorConductor(base.Conductor, metaclass=abc.ABCMeta):
try:
self._jobboard.register_entity(self.conductor)
with self._executor_factory() as executor:
self._run_until_dead(executor,
max_dispatches=max_dispatches)
self._run_until_dead(executor, max_dispatches=max_dispatches)
except StopIteration:
pass
except KeyboardInterrupt:

36
taskflow/conductors/backends/impl_nonblocking.py
View File

@@ -47,20 +47,34 @@ class NonBlockingConductor(impl_executor.ExecutorConductor):
max_workers = max_simultaneous_jobs
return futurist.ThreadPoolExecutor(max_workers=max_workers)
def __init__(self, name, jobboard,
persistence=None, engine=None,
engine_options=None, wait_timeout=None,
log=None, max_simultaneous_jobs=MAX_SIMULTANEOUS_JOBS,
executor_factory=None):
def __init__(
self,
name,
jobboard,
persistence=None,
engine=None,
engine_options=None,
wait_timeout=None,
log=None,
max_simultaneous_jobs=MAX_SIMULTANEOUS_JOBS,
executor_factory=None,
):
super().__init__(
name, jobboard,
persistence=persistence, engine=engine,
engine_options=engine_options, wait_timeout=wait_timeout,
log=log, max_simultaneous_jobs=max_simultaneous_jobs)
name,
jobboard,
persistence=persistence,
engine=engine,
engine_options=engine_options,
wait_timeout=wait_timeout,
log=log,
max_simultaneous_jobs=max_simultaneous_jobs,
)
if executor_factory is None:
self._executor_factory = self._default_executor_factory
else:
if not callable(executor_factory):
raise ValueError("Provided keyword argument 'executor_factory'"
" must be callable")
raise ValueError(
"Provided keyword argument 'executor_factory'"
" must be callable"
)
self._executor_factory = executor_factory

35
taskflow/conductors/base.py
View File

@@ -37,8 +37,14 @@ class Conductor(metaclass=abc.ABCMeta):
#: Entity kind used when creating new entity objects
ENTITY_KIND = 'conductor'
def __init__(self, name, jobboard,
persistence=None, engine=None, engine_options=None):
def __init__(
self,
name,
jobboard,
persistence=None,
engine=None,
engine_options=None,
):
self._name = name
self._jobboard = jobboard
self._engine = engine
@@ -91,9 +97,11 @@ class Conductor(metaclass=abc.ABCMeta):
flow_uuid = job.details["flow_uuid"]
flow_detail = book.find(flow_uuid)
if flow_detail is None:
raise excp.NotFound("No matching flow detail found in"
" jobs book for flow detail"
" with uuid %s" % flow_uuid)
raise excp.NotFound(
"No matching flow detail found in"
" jobs book for flow detail"
" with uuid %s" % flow_uuid
)
else:
choices = len(book)
if choices == 1:
@@ -101,8 +109,10 @@ class Conductor(metaclass=abc.ABCMeta):
elif choices == 0:
raise excp.NotFound("No flow detail(s) found in jobs book")
else:
raise excp.MultipleChoices("No matching flow detail found (%s"
" choices) in jobs book" % choices)
raise excp.MultipleChoices(
"No matching flow detail found (%s"
" choices) in jobs book" % choices
)
return flow_detail
def _engine_from_job(self, job):
@@ -116,10 +126,13 @@ class Conductor(metaclass=abc.ABCMeta):
if job.details and 'store' in job.details:
store.update(job.details["store"])
engine = engines.load_from_detail(flow_detail, store=store,
engine=self._engine,
backend=self._persistence,
**self._engine_options)
engine = engines.load_from_detail(
flow_detail,
store=store,
engine=self._engine,
backend=self._persistence,
**self._engine_options,
)
return engine
def _listeners_from_job(self, job, engine):

25
taskflow/deciders.py
View File

@@ -71,23 +71,32 @@ class Depth(misc.StrEnum):
# Nothing to do in the first place...
return desired_depth
if not isinstance(desired_depth, str):
raise TypeError("Unexpected desired depth type, string type"
" expected, not %s" % type(desired_depth))
raise TypeError(
"Unexpected desired depth type, string type"
" expected, not %s" % type(desired_depth)
)
try:
return cls(desired_depth.upper())
except ValueError:
pretty_depths = sorted([a_depth.name for a_depth in cls])
raise ValueError("Unexpected decider depth value, one of"
" %s (case-insensitive) is expected and"
" not '%s'" % (pretty_depths, desired_depth))
raise ValueError(
"Unexpected decider depth value, one of"
" %s (case-insensitive) is expected and"
" not '%s'" % (pretty_depths, desired_depth)
)
# Depth area of influence order (from greater influence to least).
#
# Order very much matters here...
_ORDERING = tuple([
Depth.ALL, Depth.FLOW, Depth.NEIGHBORS, Depth.ATOM,
])
_ORDERING = tuple(
[
Depth.ALL,
Depth.FLOW,
Depth.NEIGHBORS,
Depth.ATOM,
]
)
def pick_widest(depths):

11
taskflow/engines/__init__.py
View File

@@ -18,13 +18,14 @@ from oslo_utils import eventletutils as _eventletutils
# are highly recommended to be patched (or otherwise bad things could
# happen).
_eventletutils.warn_eventlet_not_patched(
expected_patched_modules=['time', 'thread'])
expected_patched_modules=['time', 'thread']
)
# Promote helpers to this module namespace (for easy access).
from taskflow.engines.helpers import flow_from_detail # noqa
from taskflow.engines.helpers import load # noqa
from taskflow.engines.helpers import load_from_detail # noqa
from taskflow.engines.helpers import flow_from_detail # noqa
from taskflow.engines.helpers import load # noqa
from taskflow.engines.helpers import load_from_detail # noqa
from taskflow.engines.helpers import load_from_factory # noqa
from taskflow.engines.helpers import run # noqa
from taskflow.engines.helpers import run # noqa
from taskflow.engines.helpers import save_factory_details # noqa

8
taskflow/engines/action_engine/actions/base.py
View File

@@ -26,8 +26,12 @@ class Action(metaclass=abc.ABCMeta):
"""
#: States that are expected to have a result to save...
SAVE_RESULT_STATES = (states.SUCCESS, states.FAILURE,
states.REVERTED, states.REVERT_FAILURE)
SAVE_RESULT_STATES = (
states.SUCCESS,
states.FAILURE,
states.REVERTED,
states.REVERT_FAILURE,
)
def __init__(self, storage, notifier):
self._storage = storage

10
taskflow/engines/action_engine/actions/retry.py
View File

@@ -30,13 +30,13 @@ class RetryAction(base.Action):
arguments = self._storage.fetch_mapped_args(
retry.revert_rebind,
atom_name=retry.name,
optional_args=retry.revert_optional
optional_args=retry.revert_optional,
)
else:
arguments = self._storage.fetch_mapped_args(
retry.rebind,
atom_name=retry.name,
optional_args=retry.optional
optional_args=retry.optional,
)
history = self._storage.get_retry_history(retry.name)
arguments[retry_atom.EXECUTE_REVERT_HISTORY] = history
@@ -74,7 +74,8 @@ class RetryAction(base.Action):
def schedule_execution(self, retry):
self.change_state(retry, states.RUNNING)
return self._retry_executor.execute_retry(
retry, self._get_retry_args(retry))
retry, self._get_retry_args(retry)
)
def complete_reversion(self, retry, result):
if isinstance(result, failure.Failure):
@@ -94,7 +95,8 @@ class RetryAction(base.Action):
retry_atom.REVERT_FLOW_FAILURES: self._storage.get_failures(),
}
return self._retry_executor.revert_retry(
retry, self._get_retry_args(retry, addons=arg_addons, revert=True))
retry, self._get_retry_args(retry, addons=arg_addons, revert=True)
)
def on_failure(self, retry, atom, last_failure):
self._storage.save_retry_failure(retry.name, atom.name, last_failure)

61
taskflow/engines/action_engine/actions/task.py
View File

@@ -47,11 +47,13 @@ class TaskAction(base.Action):
return False
return True
def change_state(self, task, state,
progress=None, result=base.Action.NO_RESULT):
def change_state(
self, task, state, progress=None, result=base.Action.NO_RESULT
):
old_state = self._storage.get_atom_state(task.name)
if self._is_identity_transition(old_state, state, task,
progress=progress):
if self._is_identity_transition(
old_state, state, task, progress=progress
):
# NOTE(imelnikov): ignore identity transitions in order
# to avoid extra write to storage backend and, what's
# more important, extra notifications.
@@ -85,60 +87,71 @@ class TaskAction(base.Action):
pass
else:
try:
self._storage.set_task_progress(task.name, progress,
details=details)
self._storage.set_task_progress(
task.name, progress, details=details
)
except Exception:
# Update progress callbacks should never fail, so capture and
# log the emitted exception instead of raising it.
LOG.exception("Failed setting task progress for %s to %0.3f",
task, progress)
LOG.exception(
"Failed setting task progress for %s to %0.3f",
task,
progress,
)
def schedule_execution(self, task):
self.change_state(task, states.RUNNING, progress=0.0)
arguments = self._storage.fetch_mapped_args(
task.rebind,
atom_name=task.name,
optional_args=task.optional
task.rebind, atom_name=task.name, optional_args=task.optional
)
if task.notifier.can_be_registered(task_atom.EVENT_UPDATE_PROGRESS):
progress_callback = functools.partial(self._on_update_progress,
task)
progress_callback = functools.partial(
self._on_update_progress, task
)
else:
progress_callback = None
task_uuid = self._storage.get_atom_uuid(task.name)
return self._task_executor.execute_task(
task, task_uuid, arguments,
progress_callback=progress_callback)
task, task_uuid, arguments, progress_callback=progress_callback
)
def complete_execution(self, task, result):
if isinstance(result, failure.Failure):
self.change_state(task, states.FAILURE, result=result)
else:
self.change_state(task, states.SUCCESS,
result=result, progress=1.0)
self.change_state(
task, states.SUCCESS, result=result, progress=1.0
)
def schedule_reversion(self, task):
self.change_state(task, states.REVERTING, progress=0.0)
arguments = self._storage.fetch_mapped_args(
task.revert_rebind,
atom_name=task.name,
optional_args=task.revert_optional
optional_args=task.revert_optional,
)
task_uuid = self._storage.get_atom_uuid(task.name)
task_result = self._storage.get(task.name)
failures = self._storage.get_failures()
if task.notifier.can_be_registered(task_atom.EVENT_UPDATE_PROGRESS):
progress_callback = functools.partial(self._on_update_progress,
task)
progress_callback = functools.partial(
self._on_update_progress, task
)
else:
progress_callback = None
return self._task_executor.revert_task(
task, task_uuid, arguments, task_result, failures,
progress_callback=progress_callback)
task,
task_uuid,
arguments,
task_result,
failures,
progress_callback=progress_callback,
)
def complete_reversion(self, task, result):
if isinstance(result, failure.Failure):
self.change_state(task, states.REVERT_FAILURE, result=result)
else:
self.change_state(task, states.REVERTED, progress=1.0,
result=result)
self.change_state(
task, states.REVERTED, progress=1.0, result=result
)

76
taskflow/engines/action_engine/builder.py
View File

@@ -143,9 +143,12 @@ class MachineBuilder:
def do_schedule(next_nodes):
with self._storage.lock.write_lock():
return self._scheduler.schedule(
sorted(next_nodes,
key=lambda node: getattr(node, 'priority', 0),
reverse=True))
sorted(
next_nodes,
key=lambda node: getattr(node, 'priority', 0),
reverse=True,
)
)
def iter_next_atoms(atom=None, apply_deciders=True):
# Yields and filters and tweaks the next atoms to run...
@@ -165,8 +168,10 @@ class MachineBuilder:
# that are now ready to be ran.
with self._storage.lock.write_lock():
memory.next_up.update(
iter_utils.unique_seen((self._completer.resume(),
iter_next_atoms())))
iter_utils.unique_seen(
(self._completer.resume(), iter_next_atoms())
)
)
return SCHEDULE
def game_over(old_state, new_state, event):
@@ -181,13 +186,17 @@ class MachineBuilder:
# Avoid activating the deciders, since at this point
# the engine is finishing and there will be no more further
# work done anyway...
iter_next_atoms(apply_deciders=False))
iter_next_atoms(apply_deciders=False)
)
if leftover_atoms:
# Ok we didn't finish (either reverting or executing...) so
# that means we must of been stopped at some point...
LOG.trace("Suspension determined to have been reacted to"
" since (at least) %s atoms have been left in an"
" unfinished state", leftover_atoms)
LOG.trace(
"Suspension determined to have been reacted to"
" since (at least) %s atoms have been left in an"
" unfinished state",
leftover_atoms,
)
return SUSPENDED
elif self._runtime.is_success():
return SUCCESS
@@ -239,11 +248,16 @@ class MachineBuilder:
# would suck...)
if LOG.isEnabledFor(logging.DEBUG):
intention = get_atom_intention(atom.name)
LOG.debug("Discarding failure '%s' (in response"
" to outcome '%s') under completion"
" units request during completion of"
" atom '%s' (intention is to %s)",
result, outcome, atom, intention)
LOG.debug(
"Discarding failure '%s' (in response"
" to outcome '%s') under completion"
" units request during completion of"
" atom '%s' (intention is to %s)",
result,
outcome,
atom,
intention,
)
if gather_statistics:
statistics['discarded_failures'] += 1
if gather_statistics:
@@ -256,8 +270,7 @@ class MachineBuilder:
return WAS_CANCELLED
except Exception:
memory.failures.append(failure.Failure())
LOG.exception("Engine '%s' atom post-completion"
" failed", atom)
LOG.exception("Engine '%s' atom post-completion failed", atom)
return FAILED_COMPLETING
else:
return SUCCESSFULLY_COMPLETED
@@ -286,8 +299,10 @@ class MachineBuilder:
# before we iterate over any successors or predecessors
# that we know it has been completed and saved and so on...
completion_status = complete_an_atom(fut)
if (not memory.failures
and completion_status != WAS_CANCELLED):
if (
not memory.failures
and completion_status != WAS_CANCELLED
):
atom = fut.atom
try:
more_work = set(iter_next_atoms(atom=atom))
@@ -295,12 +310,17 @@ class MachineBuilder:
memory.failures.append(failure.Failure())
LOG.exception(
"Engine '%s' atom post-completion"
" next atom searching failed", atom)
" next atom searching failed",
atom,
)
else:
next_up.update(more_work)
current_flow_state = self._storage.get_flow_state()
if (current_flow_state == st.RUNNING
and next_up and not memory.failures):
if (
current_flow_state == st.RUNNING
and next_up
and not memory.failures
):
memory.next_up.update(next_up)
return SCHEDULE
elif memory.not_done:
@@ -311,8 +331,11 @@ class MachineBuilder:
return FINISH
def on_exit(old_state, event):
LOG.trace("Exiting old state '%s' in response to event '%s'",
old_state, event)
LOG.trace(
"Exiting old state '%s' in response to event '%s'",
old_state,
event,
)
if gather_statistics:
if old_state in watches:
w = watches[old_state]
@@ -324,8 +347,11 @@ class MachineBuilder:
statistics['awaiting'] = len(memory.next_up)
def on_enter(new_state, event):
LOG.trace("Entering new state '%s' in response to event '%s'",
new_state, event)
LOG.trace(
"Entering new state '%s' in response to event '%s'",
new_state,
event,
)
if gather_statistics and new_state in watches:
watches[new_state].restart()

73
taskflow/engines/action_engine/compiler.py
View File

@@ -25,7 +25,7 @@ from taskflow.types import tree as tr
from taskflow.utils import iter_utils
from taskflow.utils import misc
from taskflow.flow import (LINK_INVARIANT, LINK_RETRY) # noqa
from taskflow.flow import LINK_INVARIANT, LINK_RETRY # noqa
LOG = logging.getLogger(__name__)
@@ -105,8 +105,9 @@ class Compilation:
def _overlap_occurrence_detector(to_graph, from_graph):
"""Returns how many nodes in 'from' graph are in 'to' graph (if any)."""
return iter_utils.count(node for node in from_graph.nodes
if node in to_graph)
return iter_utils.count(
node for node in from_graph.nodes if node in to_graph
)
def _add_update_edges(graph, nodes_from, nodes_to, attr_dict=None):
@@ -162,22 +163,30 @@ class FlowCompiler:
tree_node.add(tr.Node(flow.retry, kind=RETRY))
decomposed = {
child: self._deep_compiler_func(child, parent=tree_node)[0]
for child in flow}
for child in flow
}
decomposed_graphs = list(decomposed.values())
graph = gr.merge_graphs(graph, *decomposed_graphs,
overlap_detector=_overlap_occurrence_detector)
graph = gr.merge_graphs(
graph,
*decomposed_graphs,
overlap_detector=_overlap_occurrence_detector,
)
for u, v, attr_dict in flow.iter_links():
u_graph = decomposed[u]
v_graph = decomposed[v]
_add_update_edges(graph, u_graph.no_successors_iter(),
list(v_graph.no_predecessors_iter()),
attr_dict=attr_dict)
_add_update_edges(
graph,
u_graph.no_successors_iter(),
list(v_graph.no_predecessors_iter()),
attr_dict=attr_dict,
)
# Insert the flow(s) retry if needed, and always make sure it
# is the **immediate** successor of the flow node itself.
if flow.retry is not None:
graph.add_node(flow.retry, kind=RETRY)
_add_update_edges(graph, [flow], [flow.retry],
attr_dict={LINK_INVARIANT: True})
_add_update_edges(
graph, [flow], [flow.retry], attr_dict={LINK_INVARIANT: True}
)
for node in graph.nodes:
if node is not flow.retry and node is not flow:
graph.nodes[node].setdefault(RETRY, flow.retry)
@@ -192,10 +201,16 @@ class FlowCompiler:
# us to easily know when we have entered a flow (when running) and
# do special and/or smart things such as only traverse up to the
# start of a flow when looking for node deciders.
_add_update_edges(graph, from_nodes, [
node for node in graph.no_predecessors_iter()
if node is not flow
], attr_dict=attr_dict)
_add_update_edges(
graph,
from_nodes,
[
node
for node in graph.no_predecessors_iter()
if node is not flow
],
attr_dict=attr_dict,
)
# Connect all nodes with no successors into a special terminator
# that is used to identify the end of the flow and ensure that all
# execution traversals will traverse over this node before executing
@@ -214,10 +229,16 @@ class FlowCompiler:
# that networkx provides??
flow_term = Terminator(flow)
graph.add_node(flow_term, kind=FLOW_END, noop=True)
_add_update_edges(graph, [
node for node in graph.no_successors_iter()
if node is not flow_term
], [flow_term], attr_dict={LINK_INVARIANT: True})
_add_update_edges(
graph,
[
node
for node in graph.no_successors_iter()
if node is not flow_term
],
[flow_term],
attr_dict={LINK_INVARIANT: True},
)
return graph, tree_node
@@ -337,15 +358,19 @@ class PatternCompiler:
self._post_item_compile(item, graph, node)
return graph, node
else:
raise TypeError("Unknown object '%s' (%s) requested to compile"
% (item, type(item)))
raise TypeError(
"Unknown object '%s' (%s) requested to compile"
% (item, type(item))
)
def _pre_item_compile(self, item):
"""Called before a item is compiled; any pre-compilation actions."""
if item in self._history:
raise ValueError("Already compiled item '%s' (%s), duplicate"
" and/or recursive compiling is not"
" supported" % (item, type(item)))
raise ValueError(
"Already compiled item '%s' (%s), duplicate"
" and/or recursive compiling is not"
" supported" % (item, type(item))
)
self._history.add(item)
if LOG.isEnabledFor(logging.TRACE):
LOG.trace("%sCompiling '%s'", " " * self._level, item)

77
taskflow/engines/action_engine/completer.py
View File

@@ -58,10 +58,14 @@ class RevertAndRetry(Strategy):
self._retry = retry
def apply(self):
tweaked = self._runtime.reset_atoms([self._retry], state=None,
intention=st.RETRY)
tweaked.extend(self._runtime.reset_subgraph(self._retry, state=None,
intention=st.REVERT))
tweaked = self._runtime.reset_atoms(
[self._retry], state=None, intention=st.RETRY
)
tweaked.extend(
self._runtime.reset_subgraph(
self._retry, state=None, intention=st.REVERT
)
)
return tweaked
@@ -76,7 +80,9 @@ class RevertAll(Strategy):
def apply(self):
return self._runtime.reset_atoms(
self._runtime.iterate_nodes(co.ATOMS),
state=None, intention=st.REVERT)
state=None,
intention=st.REVERT,
)
class Revert(Strategy):
@@ -89,10 +95,14 @@ class Revert(Strategy):
self._atom = atom
def apply(self):
tweaked = self._runtime.reset_atoms([self._atom], state=None,
intention=st.REVERT)
tweaked.extend(self._runtime.reset_subgraph(self._atom, state=None,
intention=st.REVERT))
tweaked = self._runtime.reset_atoms(
[self._atom], state=None, intention=st.REVERT
)
tweaked.extend(
self._runtime.reset_subgraph(
self._atom, state=None, intention=st.REVERT
)
)
return tweaked
@@ -104,9 +114,11 @@ class Completer:
self._storage = runtime.storage
self._undefined_resolver = RevertAll(self._runtime)
self._defer_reverts = strutils.bool_from_string(
self._runtime.options.get('defer_reverts', False))
self._runtime.options.get('defer_reverts', False)
)
self._resolve = not strutils.bool_from_string(
self._runtime.options.get('never_resolve', False))
self._runtime.options.get('never_resolve', False)
)
def resume(self):
"""Resumes atoms in the contained graph.
@@ -120,14 +132,16 @@ class Completer:
attempt not previously finishing).
"""
atoms = list(self._runtime.iterate_nodes(co.ATOMS))
atom_states = self._storage.get_atoms_states(atom.name
for atom in atoms)
atom_states = self._storage.get_atoms_states(
atom.name for atom in atoms
)
if self._resolve:
for atom in atoms:
atom_state, _atom_intention = atom_states[atom.name]
if atom_state == st.FAILURE:
self._process_atom_failure(
atom, self._storage.get(atom.name))
atom, self._storage.get(atom.name)
)
for retry in self._runtime.iterate_retries(st.RETRYING):
retry_affected_atoms_it = self._runtime.retry_subflow(retry)
for atom, state, intention in retry_affected_atoms_it:
@@ -138,8 +152,11 @@ class Completer:
atom_state, _atom_intention = atom_states[atom.name]
if atom_state in (st.RUNNING, st.REVERTING):
unfinished_atoms.add(atom)
LOG.trace("Resuming atom '%s' since it was left in"
" state %s", atom, atom_state)
LOG.trace(
"Resuming atom '%s' since it was left in state %s",
atom,
atom_state,
)
return unfinished_atoms
def complete_failure(self, node, outcome, failure):
@@ -192,8 +209,10 @@ class Completer:
elif strategy == retry_atom.REVERT_ALL:
return RevertAll(self._runtime)
else:
raise ValueError("Unknown atom failure resolution"
" action/strategy '%s'" % strategy)
raise ValueError(
"Unknown atom failure resolution"
" action/strategy '%s'" % strategy
)
else:
return self._undefined_resolver
@@ -207,14 +226,24 @@ class Completer:
the failure can be worked around.
"""
resolver = self._determine_resolution(atom, failure)
LOG.debug("Applying resolver '%s' to resolve failure '%s'"
" of atom '%s'", resolver, failure, atom)
LOG.debug(
"Applying resolver '%s' to resolve failure '%s' of atom '%s'",
resolver,
failure,
atom,
)
tweaked = resolver.apply()
# Only show the tweaked node list when trace is on, otherwise
# just show the amount/count of nodes tweaks...
if LOG.isEnabledFor(logging.TRACE):
LOG.trace("Modified/tweaked %s nodes while applying"
" resolver '%s'", tweaked, resolver)
LOG.trace(
"Modified/tweaked %s nodes while applying resolver '%s'",
tweaked,
resolver,
)
else:
LOG.debug("Modified/tweaked %s nodes while applying"
" resolver '%s'", len(tweaked), resolver)
LOG.debug(
"Modified/tweaked %s nodes while applying resolver '%s'",
len(tweaked),
resolver,
)

72
taskflow/engines/action_engine/deciders.py
View File

@@ -67,23 +67,34 @@ class Decider(metaclass=abc.ABCMeta):
def _affect_all_successors(atom, runtime):
execution_graph = runtime.compilation.execution_graph
successors_iter = traversal.depth_first_iterate(
execution_graph, atom, traversal.Direction.FORWARD)
runtime.reset_atoms(itertools.chain([atom], successors_iter),
state=states.IGNORE, intention=states.IGNORE)
execution_graph, atom, traversal.Direction.FORWARD
)
runtime.reset_atoms(
itertools.chain([atom], successors_iter),
state=states.IGNORE,
intention=states.IGNORE,
)
def _affect_successor_tasks_in_same_flow(atom, runtime):
execution_graph = runtime.compilation.execution_graph
successors_iter = traversal.depth_first_iterate(
execution_graph, atom, traversal.Direction.FORWARD,
execution_graph,
atom,
traversal.Direction.FORWARD,
# Do not go through nested flows but do follow *all* tasks that
# are directly connected in this same flow (thus the reason this is
# called the same flow decider); retries are direct successors
# of flows, so they should also be not traversed through, but
# setting this explicitly ensures that.
through_flows=False, through_retries=False)
runtime.reset_atoms(itertools.chain([atom], successors_iter),
state=states.IGNORE, intention=states.IGNORE)
through_flows=False,
through_retries=False,
)
runtime.reset_atoms(
itertools.chain([atom], successors_iter),
state=states.IGNORE,
intention=states.IGNORE,
)
def _affect_atom(atom, runtime):
@@ -97,9 +108,13 @@ def _affect_direct_task_neighbors(atom, runtime):
node_data = execution_graph.nodes[node]
if node_data['kind'] == compiler.TASK:
yield node
successors_iter = _walk_neighbors()
runtime.reset_atoms(itertools.chain([atom], successors_iter),
state=states.IGNORE, intention=states.IGNORE)
runtime.reset_atoms(
itertools.chain([atom], successors_iter),
state=states.IGNORE,
intention=states.IGNORE,
)
class IgnoreDecider(Decider):
@@ -128,18 +143,23 @@ class IgnoreDecider(Decider):
# that those results can be used by the decider(s) that are
# making a decision as to pass or not pass...
states_intentions = runtime.storage.get_atoms_states(
ed.from_node.name for ed in self._edge_deciders
if ed.kind in compiler.ATOMS)
ed.from_node.name
for ed in self._edge_deciders
if ed.kind in compiler.ATOMS
)
for atom_name in states_intentions.keys():
atom_state, _atom_intention = states_intentions[atom_name]
if atom_state != states.IGNORE:
history[atom_name] = runtime.storage.get(atom_name)
for ed in self._edge_deciders:
if (ed.kind in compiler.ATOMS and
# It was an ignored atom (not included in history and
# the only way that is possible is via above loop
# skipping it...)
ed.from_node.name not in history):
if (
ed.kind in compiler.ATOMS
and
# It was an ignored atom (not included in history and
# the only way that is possible is via above loop
# skipping it...)
ed.from_node.name not in history
):
voters['ignored'].append(ed)
continue
if not ed.decider(history=history):
@@ -147,15 +167,17 @@ class IgnoreDecider(Decider):
else:
voters['run_it'].append(ed)
if LOG.isEnabledFor(logging.TRACE):
LOG.trace("Out of %s deciders there were %s 'do no run it'"
" voters, %s 'do run it' voters and %s 'ignored'"
" voters for transition to atom '%s' given history %s",
sum(len(eds) for eds in voters.values()),
list(ed.from_node.name
for ed in voters['do_not_run_it']),
list(ed.from_node.name for ed in voters['run_it']),
list(ed.from_node.name for ed in voters['ignored']),
self._atom.name, history)
LOG.trace(
"Out of %s deciders there were %s 'do no run it'"
" voters, %s 'do run it' voters and %s 'ignored'"
" voters for transition to atom '%s' given history %s",
sum(len(eds) for eds in voters.values()),
list(ed.from_node.name for ed in voters['do_not_run_it']),
list(ed.from_node.name for ed in voters['run_it']),
list(ed.from_node.name for ed in voters['ignored']),
self._atom.name,
history,
)
return voters['do_not_run_it']
def affect(self, runtime, nay_voters):

289
taskflow/engines/action_engine/engine.py
View File

@@ -55,8 +55,9 @@ def _start_stop(task_executor, retry_executor):
task_executor.stop()
def _pre_check(check_compiled=True, check_storage_ensured=True,
check_validated=True):
def _pre_check(
check_compiled=True, check_storage_ensured=True, check_validated=True
):
"""Engine state precondition checking decorator."""
def decorator(meth):
@@ -65,15 +66,21 @@ def _pre_check(check_compiled=True, check_storage_ensured=True,
@functools.wraps(meth)
def wrapper(self, *args, **kwargs):
if check_compiled and not self._compiled:
raise exc.InvalidState("Can not %s an engine which"
" has not been compiled" % do_what)
raise exc.InvalidState(
"Can not %s an engine which"
" has not been compiled" % do_what
)
if check_storage_ensured and not self._storage_ensured:
raise exc.InvalidState("Can not %s an engine"
" which has not had its storage"
" populated" % do_what)
raise exc.InvalidState(
"Can not %s an engine"
" which has not had its storage"
" populated" % do_what
)
if check_validated and not self._validated:
raise exc.InvalidState("Can not %s an engine which"
" has not been validated" % do_what)
raise exc.InvalidState(
"Can not %s an engine which"
" has not been validated" % do_what
)
return meth(self, *args, **kwargs)
return wrapper
@@ -148,8 +155,16 @@ class ActionEngine(base.Engine):
"""
IGNORABLE_STATES = frozenset(
itertools.chain([states.SCHEDULING, states.WAITING, states.RESUMING,
states.ANALYZING], builder.META_STATES))
itertools.chain(
[
states.SCHEDULING,
states.WAITING,
states.RESUMING,
states.ANALYZING,
],
builder.META_STATES,
)
)
"""
Informational states this engines internal machine yields back while
running, not useful to have the engine record but useful to provide to
@@ -175,18 +190,22 @@ class ActionEngine(base.Engine):
# or thread (this could change in the future if we desire it to).
self._retry_executor = executor.SerialRetryExecutor()
self._inject_transient = strutils.bool_from_string(
self._options.get('inject_transient', True))
self._options.get('inject_transient', True)
)
self._gather_statistics = strutils.bool_from_string(
self._options.get('gather_statistics', True))
self._options.get('gather_statistics', True)
)
self._statistics = {}
@_pre_check(check_compiled=True,
# NOTE(harlowja): We can alter the state of the
# flow without ensuring its storage is setup for
# its atoms (since this state change does not affect
# those units).
check_storage_ensured=False,
check_validated=False)
@_pre_check(
check_compiled=True,
# NOTE(harlowja): We can alter the state of the
# flow without ensuring its storage is setup for
# its atoms (since this state change does not affect
# those units).
check_storage_ensured=False,
check_validated=False,
)
def suspend(self):
self._change_state(states.SUSPENDING)
@@ -221,14 +240,18 @@ class ActionEngine(base.Engine):
the actual runtime lookup strategy, which typically will be, but is
not always different).
"""
def _scope_fetcher(atom_name):
if self._compiled:
return self._runtime.fetch_scopes_for(atom_name)
else:
return None
return storage.Storage(self._flow_detail,
backend=self._backend,
scope_fetcher=_scope_fetcher)
return storage.Storage(
self._flow_detail,
backend=self._backend,
scope_fetcher=_scope_fetcher,
)
def run(self, timeout=None):
"""Runs the engine (or die trying).
@@ -239,8 +262,9 @@ class ActionEngine(base.Engine):
"""
with fasteners.try_lock(self._lock) as was_locked:
if not was_locked:
raise exc.ExecutionFailure("Engine currently locked, please"
" try again later")
raise exc.ExecutionFailure(
"Engine currently locked, please try again later"
)
for _state in self.run_iter(timeout=timeout):
pass
@@ -272,7 +296,8 @@ class ActionEngine(base.Engine):
# are quite useful to log (and the performance of tracking this
# should be negligible).
last_transitions = collections.deque(
maxlen=max(1, self.MAX_MACHINE_STATES_RETAINED))
maxlen=max(1, self.MAX_MACHINE_STATES_RETAINED)
)
with _start_stop(self._task_executor, self._retry_executor):
self._change_state(states.RUNNING)
if self._gather_statistics:
@@ -284,8 +309,10 @@ class ActionEngine(base.Engine):
try:
closed = False
machine, memory = self._runtime.builder.build(
self._statistics, timeout=timeout,
gather_statistics=self._gather_statistics)
self._statistics,
timeout=timeout,
gather_statistics=self._gather_statistics,
)
r = runners.FiniteRunner(machine)
for transition in r.run_iter(builder.START):
last_transitions.append(transition)
@@ -317,11 +344,13 @@ class ActionEngine(base.Engine):
self.suspend()
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception("Engine execution has failed, something"
" bad must have happened (last"
" %s machine transitions were %s)",
last_transitions.maxlen,
list(last_transitions))
LOG.exception(
"Engine execution has failed, something"
" bad must have happened (last"
" %s machine transitions were %s)",
last_transitions.maxlen,
list(last_transitions),
)
self._change_state(states.FAILURE)
else:
if last_transitions:
@@ -332,8 +361,8 @@ class ActionEngine(base.Engine):
e_failures = self.storage.get_execute_failures()
r_failures = self.storage.get_revert_failures()
er_failures = itertools.chain(
e_failures.values(),
r_failures.values())
e_failures.values(), r_failures.values()
)
failure.Failure.reraise_if_any(er_failures)
finally:
if w is not None:
@@ -355,7 +384,8 @@ class ActionEngine(base.Engine):
seen.add(atom_name)
if dups:
raise exc.Duplicate(
"Atoms with duplicate names found: %s" % (sorted(dups)))
"Atoms with duplicate names found: %s" % (sorted(dups))
)
return compilation
def _change_state(self, state):
@@ -371,12 +401,12 @@ class ActionEngine(base.Engine):
def _ensure_storage(self):
"""Ensure all contained atoms exist in the storage unit."""
self.storage.ensure_atoms(
self._runtime.iterate_nodes(compiler.ATOMS))
self.storage.ensure_atoms(self._runtime.iterate_nodes(compiler.ATOMS))
for atom in self._runtime.iterate_nodes(compiler.ATOMS):
if atom.inject:
self.storage.inject_atom_args(atom.name, atom.inject,
transient=self._inject_transient)
self.storage.inject_atom_args(
atom.name, atom.inject, transient=self._inject_transient
)
@fasteners.locked
@_pre_check(check_validated=False)
@@ -387,11 +417,14 @@ class ActionEngine(base.Engine):
# by failing at validation time).
if LOG.isEnabledFor(logging.TRACE):
execution_graph = self._compilation.execution_graph
LOG.trace("Validating scoping and argument visibility for"
" execution graph with %s nodes and %s edges with"
" density %0.3f", execution_graph.number_of_nodes(),
execution_graph.number_of_edges(),
nx.density(execution_graph))
LOG.trace(
"Validating scoping and argument visibility for"
" execution graph with %s nodes and %s edges with"
" density %0.3f",
execution_graph.number_of_nodes(),
execution_graph.number_of_edges(),
nx.density(execution_graph),
)
missing = set()
# Attempt to retain a chain of what was missing (so that the final
# raised exception for the flow has the nodes that had missing
@@ -401,18 +434,25 @@ class ActionEngine(base.Engine):
missing_nodes = 0
for atom in self._runtime.iterate_nodes(compiler.ATOMS):
exec_missing = self.storage.fetch_unsatisfied_args(
atom.name, atom.rebind, optional_args=atom.optional)
atom.name, atom.rebind, optional_args=atom.optional
)
revert_missing = self.storage.fetch_unsatisfied_args(
atom.name, atom.revert_rebind,
optional_args=atom.revert_optional)
atom_missing = (('execute', exec_missing),
('revert', revert_missing))
atom.name,
atom.revert_rebind,
optional_args=atom.revert_optional,
)
atom_missing = (
('execute', exec_missing),
('revert', revert_missing),
)
for method, method_missing in atom_missing:
if method_missing:
cause = exc.MissingDependencies(atom,
sorted(method_missing),
cause=last_cause,
method=method)
cause = exc.MissingDependencies(
atom,
sorted(method_missing),
cause=last_cause,
method=method,
)
last_cause = cause
last_node = atom
missing_nodes += 1
@@ -424,9 +464,9 @@ class ActionEngine(base.Engine):
if missing_nodes == 1 and last_node is self._flow:
raise last_cause
else:
raise exc.MissingDependencies(self._flow,
sorted(missing),
cause=last_cause)
raise exc.MissingDependencies(
self._flow, sorted(missing), cause=last_cause
)
self._validated = True
@fasteners.locked
@@ -458,12 +498,14 @@ class ActionEngine(base.Engine):
if self._compiled:
return
self._compilation = self._check_compilation(self._compiler.compile())
self._runtime = runtime.Runtime(self._compilation,
self.storage,
self.atom_notifier,
self._task_executor,
self._retry_executor,
options=self._options)
self._runtime = runtime.Runtime(
self._compilation,
self.storage,
self.atom_notifier,
self._task_executor,
self._retry_executor,
options=self._options,
)
self._runtime.compile()
self._compiled = True
@@ -476,14 +518,16 @@ class SerialActionEngine(ActionEngine):
self._task_executor = executor.SerialTaskExecutor()
class _ExecutorTypeMatch(collections.namedtuple('_ExecutorTypeMatch',
['types', 'executor_cls'])):
class _ExecutorTypeMatch(
collections.namedtuple('_ExecutorTypeMatch', ['types', 'executor_cls'])
):
def matches(self, executor):
return isinstance(executor, self.types)
class _ExecutorTextMatch(collections.namedtuple('_ExecutorTextMatch',
['strings', 'executor_cls'])):
class _ExecutorTextMatch(
collections.namedtuple('_ExecutorTextMatch', ['strings', 'executor_cls'])
):
def matches(self, text):
return text.lower() in self.strings
@@ -494,51 +538,52 @@ class ParallelActionEngine(ActionEngine):
**Additional engine options:**
* ``executor``: a object that implements a :pep:`3148` compatible executor
interface; it will be used for scheduling tasks. The following
type are applicable (other unknown types passed will cause a type
error to be raised).
interface; it will be used for scheduling tasks. The following type are
applicable (other unknown types passed will cause a type error to be
raised).
========================= ===============================================
Type provided Executor used
========================= ===============================================
|cft|.ThreadPoolExecutor :class:`~.executor.ParallelThreadTaskExecutor`
|cfp|.ProcessPoolExecutor :class:`~.|pe|.ParallelProcessTaskExecutor`
|cf|._base.Executor :class:`~.executor.ParallelThreadTaskExecutor`
========================= ===============================================
========================= ==============================================
Type provided Executor used
========================= ==============================================
|cft|.ThreadPoolExecutor :class:`~.executor.ParallelThreadTaskExecutor`
|cfp|.ProcessPoolExecutor :class:`~.|pe|.ParallelProcessTaskExecutor`
|cf|._base.Executor :class:`~.executor.ParallelThreadTaskExecutor`
========================= ==============================================
* ``executor``: a string that will be used to select a :pep:`3148`
compatible executor; it will be used for scheduling tasks. The following
string are applicable (other unknown strings passed will cause a value
error to be raised).
=========================== ===============================================
String (case insensitive) Executor used
=========================== ===============================================
``process`` :class:`~.|pe|.ParallelProcessTaskExecutor`
``processes`` :class:`~.|pe|.ParallelProcessTaskExecutor`
``thread`` :class:`~.executor.ParallelThreadTaskExecutor`
``threaded`` :class:`~.executor.ParallelThreadTaskExecutor`
``threads`` :class:`~.executor.ParallelThreadTaskExecutor`
``greenthread`` :class:`~.executor.ParallelThreadTaskExecutor`
(greened version)
``greedthreaded`` :class:`~.executor.ParallelThreadTaskExecutor`
(greened version)
``greenthreads`` :class:`~.executor.ParallelThreadTaskExecutor`
(greened version)
=========================== ===============================================
=========================== ==============================================
String (case insensitive) Executor used
=========================== ==============================================
``process`` :class:`~.|pe|.ParallelProcessTaskExecutor`
``processes`` :class:`~.|pe|.ParallelProcessTaskExecutor`
``thread`` :class:`~.executor.ParallelThreadTaskExecutor`
``threaded`` :class:`~.executor.ParallelThreadTaskExecutor`
``threads`` :class:`~.executor.ParallelThreadTaskExecutor`
``greenthread`` :class:`~.executor.ParallelThreadTaskExecutor`
(greened version)
``greedthreaded`` :class:`~.executor.ParallelThreadTaskExecutor`
(greened version)
``greenthreads`` :class:`~.executor.ParallelThreadTaskExecutor`
(greened version)
=========================== ==============================================
* ``max_workers``: a integer that will affect the number of parallel
workers that are used to dispatch tasks into (this number is bounded
by the maximum parallelization your workflow can support).
* ``wait_timeout``: a float (in seconds) that will affect the
parallel process task executor (and therefore is **only** applicable when
the executor provided above is of the process variant). This number
affects how much time the process task executor waits for messages from
child processes (typically indicating they have finished or failed). A
lower number will have high granularity but *currently* involves more
polling while a higher number will involve less polling but a slower time
for an engine to notice a task has completed.
parallel process task executor (and therefore is **only** applicable
when the executor provided above is of the process variant). This
number affects how much time the process task executor waits for
messages from child processes (typically indicating they have
finished or failed). A lower number will have high granularity but
*currently* involves more polling while a higher number will involve
less polling but a slower time for an engine to notice a task has
completed.
.. |cfp| replace:: concurrent.futures.process
.. |cft| replace:: concurrent.futures.thread
@@ -552,21 +597,26 @@ String (case insensitive) Executor used
# allow for instances of that to be detected and handled correctly, instead
# of forcing everyone to use our derivatives (futurist or other)...
_executor_cls_matchers = [
_ExecutorTypeMatch((futures.ThreadPoolExecutor,),
executor.ParallelThreadTaskExecutor),
_ExecutorTypeMatch((futures.Executor,),
executor.ParallelThreadTaskExecutor),
_ExecutorTypeMatch(
(futures.ThreadPoolExecutor,), executor.ParallelThreadTaskExecutor
),
_ExecutorTypeMatch(
(futures.Executor,), executor.ParallelThreadTaskExecutor
),
]
# One of these should match when a string/text is provided for the
# 'executor' option (a mixed case equivalent is allowed since the match
# will be lower-cased before checking).
_executor_str_matchers = [
_ExecutorTextMatch(frozenset(['thread', 'threads', 'threaded']),
executor.ParallelThreadTaskExecutor),
_ExecutorTextMatch(frozenset(['greenthread', 'greenthreads',
'greenthreaded']),
executor.ParallelGreenThreadTaskExecutor),
_ExecutorTextMatch(
frozenset(['thread', 'threads', 'threaded']),
executor.ParallelThreadTaskExecutor,
),
_ExecutorTextMatch(
frozenset(['greenthread', 'greenthreads', 'greenthreaded']),
executor.ParallelGreenThreadTaskExecutor,
),
]
# Used when no executor is provided (either a string or object)...
@@ -594,9 +644,11 @@ String (case insensitive) Executor used
expected = set()
for m in cls._executor_str_matchers:
expected.update(m.strings)
raise ValueError("Unknown executor string '%s' expected"
" one of %s (or mixed case equivalent)"
% (desired_executor, list(expected)))
raise ValueError(
"Unknown executor string '%s' expected"
" one of %s (or mixed case equivalent)"
% (desired_executor, list(expected))
)
else:
executor_cls = matched_executor_cls
elif desired_executor is not None:
@@ -609,15 +661,20 @@ String (case insensitive) Executor used
expected = set()
for m in cls._executor_cls_matchers:
expected.update(m.types)
raise TypeError("Unknown executor '%s' (%s) expected an"
" instance of %s" % (desired_executor,
type(desired_executor),
list(expected)))
raise TypeError(
"Unknown executor '%s' (%s) expected an"
" instance of %s"
% (
desired_executor,
type(desired_executor),
list(expected),
)
)
else:
executor_cls = matched_executor_cls
kwargs['executor'] = desired_executor
try:
for (k, value_converter) in executor_cls.constructor_options:
for k, value_converter in executor_cls.constructor_options:
try:
kwargs[k] = value_converter(options[k])
except KeyError:

83
taskflow/engines/action_engine/executor.py
View File

@@ -42,9 +42,9 @@ def _revert_retry(retry, arguments):
def _execute_task(task, arguments, progress_callback=None):
with notifier.register_deregister(task.notifier,
ta.EVENT_UPDATE_PROGRESS,
callback=progress_callback):
with notifier.register_deregister(
task.notifier, ta.EVENT_UPDATE_PROGRESS, callback=progress_callback
):
try:
task.pre_execute()
result = task.execute(**arguments)
@@ -61,9 +61,9 @@ def _revert_task(task, arguments, result, failures, progress_callback=None):
arguments = arguments.copy()
arguments[ta.REVERT_RESULT] = result
arguments[ta.REVERT_FLOW_FAILURES] = failures
with notifier.register_deregister(task.notifier,
ta.EVENT_UPDATE_PROGRESS,
callback=progress_callback):
with notifier.register_deregister(
task.notifier, ta.EVENT_UPDATE_PROGRESS, callback=progress_callback
):
try:
task.pre_revert()
result = task.revert(**arguments)
@@ -112,13 +112,19 @@ class TaskExecutor(metaclass=abc.ABCMeta):
"""
@abc.abstractmethod
def execute_task(self, task, task_uuid, arguments,
progress_callback=None):
def execute_task(self, task, task_uuid, arguments, progress_callback=None):
"""Schedules task execution."""
@abc.abstractmethod
def revert_task(self, task, task_uuid, arguments, result, failures,
progress_callback=None):
def revert_task(
self,
task,
task_uuid,
arguments,
result,
failures,
progress_callback=None,
):
"""Schedules task reversion."""
def start(self):
@@ -141,17 +147,29 @@ class SerialTaskExecutor(TaskExecutor):
self._executor.shutdown()
def execute_task(self, task, task_uuid, arguments, progress_callback=None):
fut = self._executor.submit(_execute_task,
task, arguments,
progress_callback=progress_callback)
fut = self._executor.submit(
_execute_task, task, arguments, progress_callback=progress_callback
)
fut.atom = task
return fut
def revert_task(self, task, task_uuid, arguments, result, failures,
progress_callback=None):
fut = self._executor.submit(_revert_task,
task, arguments, result, failures,
progress_callback=progress_callback)
def revert_task(
self,
task,
task_uuid,
arguments,
result,
failures,
progress_callback=None,
):
fut = self._executor.submit(
_revert_task,
task,
arguments,
result,
failures,
progress_callback=progress_callback,
)
fut.atom = task
return fut
@@ -188,18 +206,33 @@ class ParallelTaskExecutor(TaskExecutor):
return fut
def execute_task(self, task, task_uuid, arguments, progress_callback=None):
return self._submit_task(_execute_task, task, arguments,
progress_callback=progress_callback)
return self._submit_task(
_execute_task, task, arguments, progress_callback=progress_callback
)
def revert_task(self, task, task_uuid, arguments, result, failures,
progress_callback=None):
return self._submit_task(_revert_task, task, arguments, result,
failures, progress_callback=progress_callback)
def revert_task(
self,
task,
task_uuid,
arguments,
result,
failures,
progress_callback=None,
):
return self._submit_task(
_revert_task,
task,
arguments,
result,
failures,
progress_callback=progress_callback,
)
def start(self):
if self._own_executor:
self._executor = self._create_executor(
max_workers=self._max_workers)
max_workers=self._max_workers
)
def stop(self):
if self._own_executor:

89
taskflow/engines/action_engine/runtime.py
View File

@@ -32,11 +32,12 @@ from taskflow import logging
from taskflow import states as st
from taskflow.utils import misc
from taskflow.flow import (LINK_DECIDER, LINK_DECIDER_DEPTH) # noqa
from taskflow.flow import LINK_DECIDER, LINK_DECIDER_DEPTH # noqa
# Small helper to make the edge decider tuples more easily useable...
_EdgeDecider = collections.namedtuple('_EdgeDecider',
'from_node,kind,decider,depth')
_EdgeDecider = collections.namedtuple(
'_EdgeDecider', 'from_node,kind,decider,depth'
)
LOG = logging.getLogger(__name__)
@@ -49,9 +50,15 @@ class Runtime:
action engine to run to completion.
"""
def __init__(self, compilation, storage, atom_notifier,
task_executor, retry_executor,
options=None):
def __init__(
self,
compilation,
storage,
atom_notifier,
task_executor,
retry_executor,
options=None,
):
self._atom_notifier = atom_notifier
self._task_executor = task_executor
self._retry_executor = retry_executor
@@ -65,8 +72,9 @@ class Runtime:
# This is basically a reverse breadth first exploration, with
# special logic to further traverse down flow nodes as needed...
predecessors_iter = graph.predecessors
nodes = collections.deque((u_node, atom)
for u_node in predecessors_iter(atom))
nodes = collections.deque(
(u_node, atom) for u_node in predecessors_iter(atom)
)
visited = set()
while nodes:
u_node, v_node = nodes.popleft()
@@ -77,8 +85,7 @@ class Runtime:
decider_depth = u_v_data.get(LINK_DECIDER_DEPTH)
if decider_depth is None:
decider_depth = de.Depth.ALL
yield _EdgeDecider(u_node, u_node_kind,
decider, decider_depth)
yield _EdgeDecider(u_node, u_node_kind, decider, decider_depth)
except KeyError:
pass
if u_node_kind == com.FLOW and u_node not in visited:
@@ -89,8 +96,10 @@ class Runtime:
# sure that any prior decider that was directed at this flow
# node also gets used during future decisions about this
# atom node.
nodes.extend((u_u_node, u_node)
for u_u_node in predecessors_iter(u_node))
nodes.extend(
(u_u_node, u_node)
for u_u_node in predecessors_iter(u_node)
)
def compile(self):
"""Compiles & caches frequently used execution helper objects.
@@ -102,8 +111,9 @@ class Runtime:
specific scheduler and so-on).
"""
change_state_handlers = {
com.TASK: functools.partial(self.task_action.change_state,
progress=0.0),
com.TASK: functools.partial(
self.task_action.change_state, progress=0.0
),
com.RETRY: self.retry_action.change_state,
}
schedulers = {
@@ -129,8 +139,9 @@ class Runtime:
scheduler = schedulers[node_kind]
action = actions[node_kind]
else:
raise exc.CompilationFailure("Unknown node kind '%s'"
" encountered" % node_kind)
raise exc.CompilationFailure(
"Unknown node kind '%s' encountered" % node_kind
)
metadata = {}
deciders_it = self._walk_edge_deciders(graph, node)
walker = sc.ScopeWalker(self.compilation, node, names_only=True)
@@ -140,8 +151,12 @@ class Runtime:
metadata['scheduler'] = scheduler
metadata['edge_deciders'] = tuple(deciders_it)
metadata['action'] = action
LOG.trace("Compiled %s metadata for node %s (%s)",
metadata, node.name, node_kind)
LOG.trace(
"Compiled %s metadata for node %s (%s)",
metadata,
node.name,
node_kind,
)
self._atom_cache[node.name] = metadata
# TODO(harlowja): optimize the different decider depths to avoid
# repeated full successor searching; this can be done by searching
@@ -186,15 +201,15 @@ class Runtime:
@misc.cachedproperty
def retry_action(self):
return ra.RetryAction(self._storage,
self._atom_notifier,
self._retry_executor)
return ra.RetryAction(
self._storage, self._atom_notifier, self._retry_executor
)
@misc.cachedproperty
def task_action(self):
return ta.TaskAction(self._storage,
self._atom_notifier,
self._task_executor)
return ta.TaskAction(
self._storage, self._atom_notifier, self._task_executor
)
def _fetch_atom_metadata_entry(self, atom_name, metadata_key):
return self._atom_cache[atom_name][metadata_key]
@@ -205,7 +220,8 @@ class Runtime:
# internally to the engine, and is not exposed to atoms that will
# not exist and therefore doesn't need to handle that case).
check_transition_handler = self._fetch_atom_metadata_entry(
atom.name, 'check_transition_handler')
atom.name, 'check_transition_handler'
)
return check_transition_handler(current_state, target_state)
def fetch_edge_deciders(self, atom):
@@ -250,8 +266,9 @@ class Runtime:
"""
if state:
atoms = list(self.iterate_nodes((com.RETRY,)))
atom_states = self._storage.get_atoms_states(atom.name
for atom in atoms)
atom_states = self._storage.get_atoms_states(
atom.name for atom in atoms
)
for atom in atoms:
atom_state, _atom_intention = atom_states[atom.name]
if atom_state == state:
@@ -270,8 +287,9 @@ class Runtime:
def is_success(self):
"""Checks if all atoms in the execution graph are in 'happy' state."""
atoms = list(self.iterate_nodes(com.ATOMS))
atom_states = self._storage.get_atoms_states(atom.name
for atom in atoms)
atom_states = self._storage.get_atoms_states(
atom.name for atom in atoms
)
for atom in atoms:
atom_state, _atom_intention = atom_states[atom.name]
if atom_state == st.IGNORE:
@@ -305,7 +323,8 @@ class Runtime:
tweaked.append((atom, state, intention))
if state:
change_state_handler = self._fetch_atom_metadata_entry(
atom.name, 'change_state_handler')
atom.name, 'change_state_handler'
)
change_state_handler(atom, state)
if intention:
self.storage.set_atom_intention(atom.name, intention)
@@ -313,8 +332,9 @@ class Runtime:
def reset_all(self, state=st.PENDING, intention=st.EXECUTE):
"""Resets all atoms to the given state and intention."""
return self.reset_atoms(self.iterate_nodes(com.ATOMS),
state=state, intention=intention)
return self.reset_atoms(
self.iterate_nodes(com.ATOMS), state=state, intention=intention
)
def reset_subgraph(self, atom, state=st.PENDING, intention=st.EXECUTE):
"""Resets a atoms subgraph to the given state and intention.
@@ -322,8 +342,9 @@ class Runtime:
The subgraph is contained of **all** of the atoms successors.
"""
execution_graph = self._compilation.execution_graph
atoms_it = tr.depth_first_iterate(execution_graph, atom,
tr.Direction.FORWARD)
atoms_it = tr.depth_first_iterate(
execution_graph, atom, tr.Direction.FORWARD
)
return self.reset_atoms(atoms_it, state=state, intention=intention)
def retry_subflow(self, retry):

10
taskflow/engines/action_engine/scheduler.py
View File

@@ -46,8 +46,9 @@ class RetryScheduler:
self._runtime.retry_subflow(retry)
return self._retry_action.schedule_execution(retry)
else:
raise excp.ExecutionFailure("Unknown how to schedule retry with"
" intention: %s" % intention)
raise excp.ExecutionFailure(
"Unknown how to schedule retry with intention: %s" % intention
)
class TaskScheduler:
@@ -69,8 +70,9 @@ class TaskScheduler:
elif intention == st.REVERT:
return self._task_action.schedule_reversion(task)
else:
raise excp.ExecutionFailure("Unknown how to schedule task with"
" intention: %s" % intention)
raise excp.ExecutionFailure(
"Unknown how to schedule task with intention: %s" % intention
)
class Scheduler:

25
taskflow/engines/action_engine/scopes.py
View File

@@ -32,8 +32,9 @@ class ScopeWalker:
def __init__(self, compilation, atom, names_only=False):
self._node = compilation.hierarchy.find(atom)
if self._node is None:
raise ValueError("Unable to find atom '%s' in compilation"
" hierarchy" % atom)
raise ValueError(
"Unable to find atom '%s' in compilation hierarchy" % atom
)
self._level_cache = {}
self._atom = atom
self._execution_graph = compilation.execution_graph
@@ -78,8 +79,10 @@ class ScopeWalker:
graph = self._execution_graph
if self._predecessors is None:
predecessors = {
node for node in graph.bfs_predecessors_iter(self._atom)
if graph.nodes[node]['kind'] in co.ATOMS}
node
for node in graph.bfs_predecessors_iter(self._atom)
if graph.nodes[node]['kind'] in co.ATOMS
}
self._predecessors = predecessors.copy()
else:
predecessors = self._predecessors.copy()
@@ -95,7 +98,8 @@ class ScopeWalker:
visible = []
removals = set()
atom_it = tr.depth_first_reverse_iterate(
parent, start_from_idx=last_idx)
parent, start_from_idx=last_idx
)
for atom in atom_it:
if atom in predecessors:
predecessors.remove(atom)
@@ -106,9 +110,14 @@ class ScopeWalker:
self._level_cache[lvl] = (visible, removals)
if LOG.isEnabledFor(logging.TRACE):
visible_names = [a.name for a in visible]
LOG.trace("Scope visible to '%s' (limited by parent '%s'"
" index < %s) is: %s", self._atom,
parent.item.name, last_idx, visible_names)
LOG.trace(
"Scope visible to '%s' (limited by parent '%s'"
" index < %s) is: %s",
self._atom,
parent.item.name,
last_idx,
visible_names,
)
if self._names_only:
yield [a.name for a in visible]
else:

160
taskflow/engines/action_engine/selector.py
View File

@@ -43,16 +43,22 @@ class Selector:
def iter_next_atoms(self, atom=None):
"""Iterate next atoms to run (originating from atom or all atoms)."""
if atom is None:
return iter_utils.unique_seen((self._browse_atoms_for_execute(),
self._browse_atoms_for_revert()),
seen_selector=operator.itemgetter(0))
return iter_utils.unique_seen(
(
self._browse_atoms_for_execute(),
self._browse_atoms_for_revert(),
),
seen_selector=operator.itemgetter(0),
)
state = self._storage.get_atom_state(atom.name)
intention = self._storage.get_atom_intention(atom.name)
if state == st.SUCCESS:
if intention == st.REVERT:
return iter([
(atom, deciders.NoOpDecider()),
])
return iter(
[
(atom, deciders.NoOpDecider()),
]
)
elif intention == st.EXECUTE:
return self._browse_atoms_for_execute(atom=atom)
else:
@@ -82,7 +88,8 @@ class Selector:
# problematic to determine as top levels can have their deciders
# applied **after** going deeper).
atom_it = traversal.breadth_first_iterate(
self._execution_graph, atom, traversal.Direction.FORWARD)
self._execution_graph, atom, traversal.Direction.FORWARD
)
for atom in atom_it:
is_ready, late_decider = self._get_maybe_ready_for_execute(atom)
if is_ready:
@@ -100,22 +107,32 @@ class Selector:
atom_it = self._runtime.iterate_nodes(co.ATOMS)
else:
atom_it = traversal.breadth_first_iterate(
self._execution_graph, atom, traversal.Direction.BACKWARD,
self._execution_graph,
atom,
traversal.Direction.BACKWARD,
# Stop at the retry boundary (as retries 'control' there
# surronding atoms, and we don't want to back track over
# them so that they can correctly affect there associated
# atoms); we do though need to jump through all tasks since
# if a predecessor Y was ignored and a predecessor Z before Y
# was not it should be eligible to now revert...
through_retries=False)
through_retries=False,
)
for atom in atom_it:
is_ready, late_decider = self._get_maybe_ready_for_revert(atom)
if is_ready:
yield (atom, late_decider)
def _get_maybe_ready(self, atom, transition_to, allowed_intentions,
connected_fetcher, ready_checker,
decider_fetcher, for_what="?"):
def _get_maybe_ready(
self,
atom,
transition_to,
allowed_intentions,
connected_fetcher,
ready_checker,
decider_fetcher,
for_what="?",
):
def iter_connected_states():
# Lazily iterate over connected states so that ready checkers
# can stop early (vs having to consume and check all the
@@ -126,6 +143,7 @@ class Selector:
# to avoid two calls into storage).
atom_states = self._storage.get_atoms_states([atom.name])
yield (atom, atom_states[atom.name])
# NOTE(harlowja): How this works is the following...
#
# 1. First check if the current atom can even transition to the
@@ -144,18 +162,29 @@ class Selector:
# which can (if it desires) affect this ready result (but does
# so right before the atom is about to be scheduled).
state = self._storage.get_atom_state(atom.name)
ok_to_transition = self._runtime.check_atom_transition(atom, state,
transition_to)
ok_to_transition = self._runtime.check_atom_transition(
atom, state, transition_to
)
if not ok_to_transition:
LOG.trace("Atom '%s' is not ready to %s since it can not"
" transition to %s from its current state %s",
atom, for_what, transition_to, state)
LOG.trace(
"Atom '%s' is not ready to %s since it can not"
" transition to %s from its current state %s",
atom,
for_what,
transition_to,
state,
)
return (False, None)
intention = self._storage.get_atom_intention(atom.name)
if intention not in allowed_intentions:
LOG.trace("Atom '%s' is not ready to %s since its current"
" intention %s is not in allowed intentions %s",
atom, for_what, intention, allowed_intentions)
LOG.trace(
"Atom '%s' is not ready to %s since its current"
" intention %s is not in allowed intentions %s",
atom,
for_what,
intention,
allowed_intentions,
)
return (False, None)
ok_to_run = ready_checker(iter_connected_states())
if not ok_to_run:
@@ -165,62 +194,91 @@ class Selector:
def _get_maybe_ready_for_execute(self, atom):
"""Returns if an atom is *likely* ready to be executed."""
def ready_checker(pred_connected_it):
for pred in pred_connected_it:
pred_atom, (pred_atom_state, pred_atom_intention) = pred
if (pred_atom_state in (st.SUCCESS, st.IGNORE) and
pred_atom_intention in (st.EXECUTE, st.IGNORE)):
if pred_atom_state in (
st.SUCCESS,
st.IGNORE,
) and pred_atom_intention in (st.EXECUTE, st.IGNORE):
continue
LOG.trace("Unable to begin to execute since predecessor"
" atom '%s' is in state %s with intention %s",
pred_atom, pred_atom_state, pred_atom_intention)
LOG.trace(
"Unable to begin to execute since predecessor"
" atom '%s' is in state %s with intention %s",
pred_atom,
pred_atom_state,
pred_atom_intention,
)
return False
LOG.trace("Able to let '%s' execute", atom)
return True
decider_fetcher = lambda: \
deciders.IgnoreDecider(
atom, self._runtime.fetch_edge_deciders(atom))
connected_fetcher = lambda: \
traversal.depth_first_iterate(self._execution_graph, atom,
# Whether the desired atom
# can execute is dependent on its
# predecessors outcomes (thus why
# we look backwards).
traversal.Direction.BACKWARD)
decider_fetcher = lambda: deciders.IgnoreDecider(
atom, self._runtime.fetch_edge_deciders(atom)
)
connected_fetcher = lambda: traversal.depth_first_iterate(
self._execution_graph,
atom,
# Whether the desired atom
# can execute is dependent on its
# predecessors outcomes (thus why
# we look backwards).
traversal.Direction.BACKWARD,
)
# If this atoms current state is able to be transitioned to RUNNING
# and its intention is to EXECUTE and all of its predecessors executed
# successfully or were ignored then this atom is ready to execute.
LOG.trace("Checking if '%s' is ready to execute", atom)
return self._get_maybe_ready(atom, st.RUNNING, [st.EXECUTE],
connected_fetcher, ready_checker,
decider_fetcher, for_what='execute')
return self._get_maybe_ready(
atom,
st.RUNNING,
[st.EXECUTE],
connected_fetcher,
ready_checker,
decider_fetcher,
for_what='execute',
)
def _get_maybe_ready_for_revert(self, atom):
"""Returns if an atom is *likely* ready to be reverted."""
def ready_checker(succ_connected_it):
for succ in succ_connected_it:
succ_atom, (succ_atom_state, _succ_atom_intention) = succ
if succ_atom_state not in (st.PENDING, st.REVERTED, st.IGNORE):
LOG.trace("Unable to begin to revert since successor"
" atom '%s' is in state %s", succ_atom,
succ_atom_state)
LOG.trace(
"Unable to begin to revert since successor"
" atom '%s' is in state %s",
succ_atom,
succ_atom_state,
)
return False
LOG.trace("Able to let '%s' revert", atom)
return True
noop_decider = deciders.NoOpDecider()
connected_fetcher = lambda: \
traversal.depth_first_iterate(self._execution_graph, atom,
# Whether the desired atom
# can revert is dependent on its
# successors states (thus why we
# look forwards).
traversal.Direction.FORWARD)
connected_fetcher = lambda: traversal.depth_first_iterate(
self._execution_graph,
atom,
# Whether the desired atom
# can revert is dependent on its
# successors states (thus why we
# look forwards).
traversal.Direction.FORWARD,
)
decider_fetcher = lambda: noop_decider
# If this atoms current state is able to be transitioned to REVERTING
# and its intention is either REVERT or RETRY and all of its
# successors are either PENDING or REVERTED then this atom is ready
# to revert.
LOG.trace("Checking if '%s' is ready to revert", atom)
return self._get_maybe_ready(atom, st.REVERTING, [st.REVERT, st.RETRY],
connected_fetcher, ready_checker,
decider_fetcher, for_what='revert')
return self._get_maybe_ready(
atom,
st.REVERTING,
[st.REVERT, st.RETRY],
connected_fetcher,
ready_checker,
decider_fetcher,
for_what='revert',
)

53
taskflow/engines/action_engine/traversal.py
View File

@@ -28,9 +28,14 @@ class Direction(enum.Enum):
BACKWARD = 2
def _extract_connectors(execution_graph, starting_node, direction,
through_flows=True, through_retries=True,
through_tasks=True):
def _extract_connectors(
execution_graph,
starting_node,
direction,
through_flows=True,
through_retries=True,
through_tasks=True,
):
if direction == Direction.FORWARD:
connected_iter = execution_graph.successors
else:
@@ -46,9 +51,14 @@ def _extract_connectors(execution_graph, starting_node, direction,
return connected_iter(starting_node), connected_to_functors
def breadth_first_iterate(execution_graph, starting_node, direction,
through_flows=True, through_retries=True,
through_tasks=True):
def breadth_first_iterate(
execution_graph,
starting_node,
direction,
through_flows=True,
through_retries=True,
through_tasks=True,
):
"""Iterates connected nodes in execution graph (from starting node).
Does so in a breadth first manner.
@@ -56,9 +66,13 @@ def breadth_first_iterate(execution_graph, starting_node, direction,
Jumps over nodes with ``noop`` attribute (does not yield them back).
"""
initial_nodes_iter, connected_to_functors = _extract_connectors(
execution_graph, starting_node, direction,
through_flows=through_flows, through_retries=through_retries,
through_tasks=through_tasks)
execution_graph,
starting_node,
direction,
through_flows=through_flows,
through_retries=through_retries,
through_tasks=through_tasks,
)
q = collections.deque(initial_nodes_iter)
visited_nodes = set()
while q:
@@ -79,9 +93,14 @@ def breadth_first_iterate(execution_graph, starting_node, direction,
q.extend(connected_to_functor(node))
def depth_first_iterate(execution_graph, starting_node, direction,
through_flows=True, through_retries=True,
through_tasks=True):
def depth_first_iterate(
execution_graph,
starting_node,
direction,
through_flows=True,
through_retries=True,
through_tasks=True,
):
"""Iterates connected nodes in execution graph (from starting node).
Does so in a depth first manner.
@@ -89,9 +108,13 @@ def depth_first_iterate(execution_graph, starting_node, direction,
Jumps over nodes with ``noop`` attribute (does not yield them back).
"""
initial_nodes_iter, connected_to_functors = _extract_connectors(
execution_graph, starting_node, direction,
through_flows=through_flows, through_retries=through_retries,
through_tasks=through_tasks)
execution_graph,
starting_node,
direction,
through_flows=through_flows,
through_retries=through_retries,
through_tasks=through_tasks,
)
stack = list(initial_nodes_iter)
visited_nodes = set()
while stack:

144
taskflow/engines/helpers.py
View File

@@ -60,8 +60,9 @@ def _fetch_factory(factory_name):
try:
return importutils.import_class(factory_name)
except (ImportError, ValueError) as e:
raise ImportError("Could not import factory %r: %s"
% (factory_name, e))
raise ImportError(
"Could not import factory %r: %s" % (factory_name, e)
)
def _fetch_validate_factory(flow_factory):
@@ -73,16 +74,25 @@ def _fetch_validate_factory(flow_factory):
factory_name = reflection.get_callable_name(flow_factory)
try:
reimported = _fetch_factory(factory_name)
assert reimported == factory_fun
assert reimported == factory_fun # noqa: S101
except (ImportError, AssertionError):
raise ValueError('Flow factory %r is not reimportable by name %s'
% (factory_fun, factory_name))
raise ValueError(
'Flow factory %r is not reimportable by name %s'
% (factory_fun, factory_name)
)
return (factory_name, factory_fun)
def load(flow, store=None, flow_detail=None, book=None,
backend=None, namespace=ENGINES_NAMESPACE,
engine=ENGINE_DEFAULT, **kwargs):
def load(
flow,
store=None,
flow_detail=None,
book=None,
backend=None,
namespace=ENGINES_NAMESPACE,
engine=ENGINE_DEFAULT,
**kwargs,
):
"""Load a flow into an engine.
This function creates and prepares an engine to run the provided flow. All
@@ -122,15 +132,18 @@ def load(flow, store=None, flow_detail=None, book=None,
backend = p_backends.fetch(backend)
if flow_detail is None:
flow_detail = p_utils.create_flow_detail(flow, book=book,
backend=backend)
flow_detail = p_utils.create_flow_detail(
flow, book=book, backend=backend
)
LOG.debug('Looking for %r engine driver in %r', kind, namespace)
try:
mgr = stevedore.driver.DriverManager(
namespace, kind,
namespace,
kind,
invoke_on_load=True,
invoke_args=(flow, flow_detail, backend, options))
invoke_args=(flow, flow_detail, backend, options),
)
engine = mgr.driver
except RuntimeError as e:
raise exc.NotFound("Could not find engine '%s'" % (kind), e)
@@ -140,9 +153,16 @@ def load(flow, store=None, flow_detail=None, book=None,
return engine
def run(flow, store=None, flow_detail=None, book=None,
backend=None, namespace=ENGINES_NAMESPACE,
engine=ENGINE_DEFAULT, **kwargs):
def run(
flow,
store=None,
flow_detail=None,
book=None,
backend=None,
namespace=ENGINES_NAMESPACE,
engine=ENGINE_DEFAULT,
**kwargs,
):
"""Run the flow.
This function loads the flow into an engine (with the :func:`load() <load>`
@@ -153,16 +173,23 @@ def run(flow, store=None, flow_detail=None, book=None,
:returns: dictionary of all named
results (see :py:meth:`~.taskflow.storage.Storage.fetch_all`)
"""
engine = load(flow, store=store, flow_detail=flow_detail, book=book,
backend=backend, namespace=namespace,
engine=engine, **kwargs)
engine = load(
flow,
store=store,
flow_detail=flow_detail,
book=book,
backend=backend,
namespace=namespace,
engine=engine,
**kwargs,
)
engine.run()
return engine.storage.fetch_all()
def save_factory_details(flow_detail,
flow_factory, factory_args, factory_kwargs,
backend=None):
def save_factory_details(
flow_detail, flow_factory, factory_args, factory_kwargs, backend=None
):
"""Saves the given factories reimportable attributes into the flow detail.
This function saves the factory name, arguments, and keyword arguments
@@ -198,10 +225,17 @@ def save_factory_details(flow_detail,
conn.update_flow_details(flow_detail)
def load_from_factory(flow_factory, factory_args=None, factory_kwargs=None,
store=None, book=None, backend=None,
namespace=ENGINES_NAMESPACE, engine=ENGINE_DEFAULT,
**kwargs):
def load_from_factory(
flow_factory,
factory_args=None,
factory_kwargs=None,
store=None,
book=None,
backend=None,
namespace=ENGINES_NAMESPACE,
engine=ENGINE_DEFAULT,
**kwargs,
):
"""Loads a flow from a factory function into an engine.
Gets flow factory function (or name of it) and creates flow with
@@ -227,12 +261,23 @@ def load_from_factory(flow_factory, factory_args=None, factory_kwargs=None,
if isinstance(backend, dict):
backend = p_backends.fetch(backend)
flow_detail = p_utils.create_flow_detail(flow, book=book, backend=backend)
save_factory_details(flow_detail,
flow_factory, factory_args, factory_kwargs,
backend=backend)
return load(flow=flow, store=store, flow_detail=flow_detail, book=book,
backend=backend, namespace=namespace,
engine=engine, **kwargs)
save_factory_details(
flow_detail,
flow_factory,
factory_args,
factory_kwargs,
backend=backend,
)
return load(
flow=flow,
store=store,
flow_detail=flow_detail,
book=book,
backend=backend,
namespace=namespace,
engine=engine,
**kwargs,
)
def flow_from_detail(flow_detail):
@@ -247,24 +292,33 @@ def flow_from_detail(flow_detail):
try:
factory_data = flow_detail.meta['factory']
except (KeyError, AttributeError, TypeError):
raise ValueError('Cannot reconstruct flow %s %s: '
'no factory information saved.'
% (flow_detail.name, flow_detail.uuid))
raise ValueError(
'Cannot reconstruct flow %s %s: '
'no factory information saved.'
% (flow_detail.name, flow_detail.uuid)
)
try:
factory_fun = _fetch_factory(factory_data['name'])
except (KeyError, ImportError):
raise ImportError('Could not import factory for flow %s %s'
% (flow_detail.name, flow_detail.uuid))
raise ImportError(
'Could not import factory for flow %s %s'
% (flow_detail.name, flow_detail.uuid)
)
args = factory_data.get('args', ())
kwargs = factory_data.get('kwargs', {})
return factory_fun(*args, **kwargs)
def load_from_detail(flow_detail, store=None, backend=None,
namespace=ENGINES_NAMESPACE, engine=ENGINE_DEFAULT,
**kwargs):
def load_from_detail(
flow_detail,
store=None,
backend=None,
namespace=ENGINES_NAMESPACE,
engine=ENGINE_DEFAULT,
**kwargs,
):
"""Reloads an engine previously saved.
This reloads the flow using the
@@ -278,6 +332,12 @@ def load_from_detail(flow_detail, store=None, backend=None,
:returns: engine
"""
flow = flow_from_detail(flow_detail)
return load(flow, flow_detail=flow_detail,
store=store, backend=backend,
namespace=namespace, engine=engine, **kwargs)
return load(
flow,
flow_detail=flow_detail,
store=store,
backend=backend,
namespace=namespace,
engine=engine,
**kwargs,
)

72
taskflow/engines/worker_based/dispatcher.py
View File

@@ -100,9 +100,13 @@ class TypeDispatcher:
if cb(data, message):
requeue_votes += 1
except Exception:
LOG.exception("Failed calling requeue filter %s '%s' to"
" determine if message %r should be requeued.",
i + 1, cb, message.delivery_tag)
LOG.exception(
"Failed calling requeue filter %s '%s' to"
" determine if message %r should be requeued.",
i + 1,
cb,
message.delivery_tag,
)
return requeue_votes
def _requeue_log_error(self, message, errors):
@@ -114,52 +118,72 @@ class TypeDispatcher:
# This was taken from how kombu is formatting its messages
# when its reject_log_error or ack_log_error functions are
# used so that we have a similar error format for requeuing.
LOG.critical("Couldn't requeue %r, reason:%r",
message.delivery_tag, exc, exc_info=True)
LOG.critical(
"Couldn't requeue %r, reason:%r",
message.delivery_tag,
exc,
exc_info=True,
)
else:
LOG.debug("Message '%s' was requeued.", ku.DelayedPretty(message))
def _process_message(self, data, message, message_type):
handler = self._type_handlers.get(message_type)
if handler is None:
message.reject_log_error(logger=LOG,
errors=(kombu_exc.MessageStateError,))
LOG.warning("Unexpected message type: '%s' in message"
" '%s'", message_type, ku.DelayedPretty(message))
message.reject_log_error(
logger=LOG, errors=(kombu_exc.MessageStateError,)
)
LOG.warning(
"Unexpected message type: '%s' in message '%s'",
message_type,
ku.DelayedPretty(message),
)
else:
if handler.validator is not None:
try:
handler.validator(data)
except excp.InvalidFormat as e:
message.reject_log_error(
logger=LOG, errors=(kombu_exc.MessageStateError,))
LOG.warning("Message '%s' (%s) was rejected due to it"
" being in an invalid format: %s",
ku.DelayedPretty(message), message_type, e)
logger=LOG, errors=(kombu_exc.MessageStateError,)
)
LOG.warning(
"Message '%s' (%s) was rejected due to it"
" being in an invalid format: %s",
ku.DelayedPretty(message),
message_type,
e,
)
return
message.ack_log_error(logger=LOG,
errors=(kombu_exc.MessageStateError,))
message.ack_log_error(
logger=LOG, errors=(kombu_exc.MessageStateError,)
)
if message.acknowledged:
LOG.debug("Message '%s' was acknowledged.",
ku.DelayedPretty(message))
LOG.debug(
"Message '%s' was acknowledged.", ku.DelayedPretty(message)
)
handler.process_message(data, message)
else:
message.reject_log_error(logger=LOG,
errors=(kombu_exc.MessageStateError,))
message.reject_log_error(
logger=LOG, errors=(kombu_exc.MessageStateError,)
)
def on_message(self, data, message):
"""This method is called on incoming messages."""
LOG.debug("Received message '%s'", ku.DelayedPretty(message))
if self._collect_requeue_votes(data, message):
self._requeue_log_error(message,
errors=(kombu_exc.MessageStateError,))
self._requeue_log_error(
message, errors=(kombu_exc.MessageStateError,)
)
else:
try:
message_type = message.properties['type']
except KeyError:
message.reject_log_error(
logger=LOG, errors=(kombu_exc.MessageStateError,))
LOG.warning("The 'type' message property is missing"
" in message '%s'", ku.DelayedPretty(message))
logger=LOG, errors=(kombu_exc.MessageStateError,)
)
LOG.warning(
"The 'type' message property is missing in message '%s'",
ku.DelayedPretty(message),
)
else:
self._process_message(data, message, message_type)

21
taskflow/engines/worker_based/engine.py
View File

@@ -54,17 +54,20 @@ class WorkerBasedActionEngine(engine.ActionEngine):
super().__init__(flow, flow_detail, backend, options)
# This ensures that any provided executor will be validated before
# we get to far in the compilation/execution pipeline...
self._task_executor = self._fetch_task_executor(self._options,
self._flow_detail)
self._task_executor = self._fetch_task_executor(
self._options, self._flow_detail
)
@classmethod
def _fetch_task_executor(cls, options, flow_detail):
try:
e = options['executor']
if not isinstance(e, executor.WorkerTaskExecutor):
raise TypeError("Expected an instance of type '%s' instead of"
" type '%s' for 'executor' option"
% (executor.WorkerTaskExecutor, type(e)))
raise TypeError(
"Expected an instance of type '%s' instead of"
" type '%s' for 'executor' option"
% (executor.WorkerTaskExecutor, type(e))
)
return e
except KeyError:
return executor.WorkerTaskExecutor(
@@ -75,8 +78,8 @@ class WorkerBasedActionEngine(engine.ActionEngine):
topics=options.get('topics', []),
transport=options.get('transport'),
transport_options=options.get('transport_options'),
transition_timeout=options.get('transition_timeout',
pr.REQUEST_TIMEOUT),
worker_expiry=options.get('worker_expiry',
pr.EXPIRES_AFTER),
transition_timeout=options.get(
'transition_timeout', pr.REQUEST_TIMEOUT
),
worker_expiry=options.get('worker_expiry', pr.EXPIRES_AFTER),
)

215
taskflow/engines/worker_based/executor.py
View File

@@ -35,35 +35,53 @@ LOG = logging.getLogger(__name__)
class WorkerTaskExecutor(executor.TaskExecutor):
"""Executes tasks on remote workers."""
def __init__(self, uuid, exchange, topics,
transition_timeout=pr.REQUEST_TIMEOUT,
url=None, transport=None, transport_options=None,
retry_options=None, worker_expiry=pr.EXPIRES_AFTER):
def __init__(
self,
uuid,
exchange,
topics,
transition_timeout=pr.REQUEST_TIMEOUT,
url=None,
transport=None,
transport_options=None,
retry_options=None,
worker_expiry=pr.EXPIRES_AFTER,
):
self._uuid = uuid
self._ongoing_requests = {}
self._ongoing_requests_lock = threading.RLock()
self._transition_timeout = transition_timeout
self._proxy = proxy.Proxy(uuid, exchange,
on_wait=self._on_wait, url=url,
transport=transport,
transport_options=transport_options,
retry_options=retry_options)
self._proxy = proxy.Proxy(
uuid,
exchange,
on_wait=self._on_wait,
url=url,
transport=transport,
transport_options=transport_options,
retry_options=retry_options,
)
# NOTE(harlowja): This is the most simplest finder impl. that
# doesn't have external dependencies (outside of what this engine
# already requires); it though does create periodic 'polling' traffic
# to workers to 'learn' of the tasks they can perform (and requires
# pre-existing knowledge of the topics those workers are on to gather
# and update this information).
self._finder = wt.ProxyWorkerFinder(uuid, self._proxy, topics,
worker_expiry=worker_expiry)
self._proxy.dispatcher.type_handlers.update({
pr.RESPONSE: dispatcher.Handler(self._process_response,
validator=pr.Response.validate),
pr.NOTIFY: dispatcher.Handler(
self._finder.process_response,
validator=functools.partial(pr.Notify.validate,
response=True)),
})
self._finder = wt.ProxyWorkerFinder(
uuid, self._proxy, topics, worker_expiry=worker_expiry
)
self._proxy.dispatcher.type_handlers.update(
{
pr.RESPONSE: dispatcher.Handler(
self._process_response, validator=pr.Response.validate
),
pr.NOTIFY: dispatcher.Handler(
self._finder.process_response,
validator=functools.partial(
pr.Notify.validate, response=True
),
),
}
)
# Thread that will run the message dispatching (and periodically
# call the on_wait callback to do various things) loop...
self._helper = None
@@ -73,20 +91,27 @@ class WorkerTaskExecutor(executor.TaskExecutor):
def _process_response(self, response, message):
"""Process response from remote side."""
LOG.debug("Started processing response message '%s'",
ku.DelayedPretty(message))
LOG.debug(
"Started processing response message '%s'",
ku.DelayedPretty(message),
)
try:
request_uuid = message.properties['correlation_id']
except KeyError:
LOG.warning("The 'correlation_id' message property is"
" missing in message '%s'",
ku.DelayedPretty(message))
LOG.warning(
"The 'correlation_id' message property is"
" missing in message '%s'",
ku.DelayedPretty(message),
)
else:
request = self._ongoing_requests.get(request_uuid)
if request is not None:
response = pr.Response.from_dict(response)
LOG.debug("Extracted response '%s' and matched it to"
" request '%s'", response, request)
LOG.debug(
"Extracted response '%s' and matched it to request '%s'",
response,
request,
)
if response.state == pr.RUNNING:
request.transition_and_log_error(pr.RUNNING, logger=LOG)
elif response.state == pr.EVENT:
@@ -98,14 +123,16 @@ class WorkerTaskExecutor(executor.TaskExecutor):
details = response.data['details']
request.task.notifier.notify(event_type, details)
elif response.state in (pr.FAILURE, pr.SUCCESS):
if request.transition_and_log_error(response.state,
logger=LOG):
if request.transition_and_log_error(
response.state, logger=LOG
):
with self._ongoing_requests_lock:
del self._ongoing_requests[request.uuid]
request.set_result(result=response.data['result'])
else:
LOG.warning("Unexpected response status '%s'",
response.state)
LOG.warning(
"Unexpected response status '%s'", response.state
)
else:
LOG.debug("Request with id='%s' not found", request_uuid)
@@ -126,7 +153,8 @@ class WorkerTaskExecutor(executor.TaskExecutor):
raise exc.RequestTimeout(
"Request '%s' has expired after waiting for %0.2f"
" seconds for it to transition out of (%s) states"
% (request, request_age, ", ".join(pr.WAITING_STATES)))
% (request, request_age, ", ".join(pr.WAITING_STATES))
)
except exc.RequestTimeout:
with misc.capture_failure() as failure:
LOG.debug(failure.exception_str)
@@ -169,9 +197,9 @@ class WorkerTaskExecutor(executor.TaskExecutor):
while waiting_requests:
_request_uuid, request = waiting_requests.popitem()
worker = finder.get_worker_for_task(request.task)
if (worker is not None and
request.transition_and_log_error(pr.PENDING,
logger=LOG)):
if worker is not None and request.transition_and_log_error(
pr.PENDING, logger=LOG
):
self._publish_request(request, worker)
self._messages_processed['finder'] = new_messages_processed
@@ -187,20 +215,36 @@ class WorkerTaskExecutor(executor.TaskExecutor):
# a worker located).
self._clean()
def _submit_task(self, task, task_uuid, action, arguments,
progress_callback=None, result=pr.NO_RESULT,
failures=None):
def _submit_task(
self,
task,
task_uuid,
action,
arguments,
progress_callback=None,
result=pr.NO_RESULT,
failures=None,
):
"""Submit task request to a worker."""
request = pr.Request(task, task_uuid, action, arguments,
timeout=self._transition_timeout,
result=result, failures=failures)
request = pr.Request(
task,
task_uuid,
action,
arguments,
timeout=self._transition_timeout,
result=result,
failures=failures,
)
# Register the callback, so that we can proxy the progress correctly.
if (progress_callback is not None and
task.notifier.can_be_registered(EVENT_UPDATE_PROGRESS)):
if progress_callback is not None and task.notifier.can_be_registered(
EVENT_UPDATE_PROGRESS
):
task.notifier.register(EVENT_UPDATE_PROGRESS, progress_callback)
request.future.add_done_callback(
lambda _fut: task.notifier.deregister(EVENT_UPDATE_PROGRESS,
progress_callback))
lambda _fut: task.notifier.deregister(
EVENT_UPDATE_PROGRESS, progress_callback
)
)
# Get task's worker and publish request if worker was found.
worker = self._finder.get_worker_for_task(task)
if worker is not None:
@@ -209,42 +253,75 @@ class WorkerTaskExecutor(executor.TaskExecutor):
self._ongoing_requests[request.uuid] = request
self._publish_request(request, worker)
else:
LOG.debug("Delaying submission of '%s', no currently known"
" worker/s available to process it", request)
LOG.debug(
"Delaying submission of '%s', no currently known"
" worker/s available to process it",
request,
)
with self._ongoing_requests_lock:
self._ongoing_requests[request.uuid] = request
return request.future
def _publish_request(self, request, worker):
"""Publish request to a given topic."""
LOG.debug("Submitting execution of '%s' to worker '%s' (expecting"
" response identified by reply_to=%s and"
" correlation_id=%s) - waited %0.3f seconds to"
" get published", request, worker, self._uuid,
request.uuid, timeutils.now() - request.created_on)
LOG.debug(
"Submitting execution of '%s' to worker '%s' (expecting"
" response identified by reply_to=%s and"
" correlation_id=%s) - waited %0.3f seconds to"
" get published",
request,
worker,
self._uuid,
request.uuid,
timeutils.now() - request.created_on,
)
try:
self._proxy.publish(request, worker.topic,
reply_to=self._uuid,
correlation_id=request.uuid)
self._proxy.publish(
request,
worker.topic,
reply_to=self._uuid,
correlation_id=request.uuid,
)
except Exception:
with misc.capture_failure() as failure:
LOG.critical("Failed to submit '%s' (transitioning it to"
" %s)", request, pr.FAILURE, exc_info=True)
LOG.critical(
"Failed to submit '%s' (transitioning it to %s)",
request,
pr.FAILURE,
exc_info=True,
)
if request.transition_and_log_error(pr.FAILURE, logger=LOG):
with self._ongoing_requests_lock:
del self._ongoing_requests[request.uuid]
request.set_result(failure)
def execute_task(self, task, task_uuid, arguments,
progress_callback=None):
return self._submit_task(task, task_uuid, pr.EXECUTE, arguments,
progress_callback=progress_callback)
def execute_task(self, task, task_uuid, arguments, progress_callback=None):
return self._submit_task(
task,
task_uuid,
pr.EXECUTE,
arguments,
progress_callback=progress_callback,
)
def revert_task(self, task, task_uuid, arguments, result, failures,
progress_callback=None):
return self._submit_task(task, task_uuid, pr.REVERT, arguments,
result=result, failures=failures,
progress_callback=progress_callback)
def revert_task(
self,
task,
task_uuid,
arguments,
result,
failures,
progress_callback=None,
):
return self._submit_task(
task,
task_uuid,
pr.REVERT,
arguments,
result=result,
failures=failures,
progress_callback=progress_callback,
)
def wait_for_workers(self, workers=1, timeout=None):
"""Waits for geq workers to notify they are ready to do work.
@@ -255,14 +332,14 @@ class WorkerTaskExecutor(executor.TaskExecutor):
return how many workers are still needed, otherwise it will
return zero.
"""
return self._finder.wait_for_workers(workers=workers,
timeout=timeout)
return self._finder.wait_for_workers(workers=workers, timeout=timeout)
def start(self):
"""Starts message processing thread."""
if self._helper is not None:
raise RuntimeError("Worker executor must be stopped before"
" it can be started")
raise RuntimeError(
"Worker executor must be stopped before it can be started"
)
self._helper = tu.daemon_thread(self._proxy.start)
self._helper.start()
self._proxy.wait()

100
taskflow/engines/worker_based/protocol.py
View File

@@ -53,10 +53,7 @@ EXECUTE = 'execute'
REVERT = 'revert'
# Remote task action to event map.
ACTION_TO_EVENT = {
EXECUTE: executor.EXECUTED,
REVERT: executor.REVERTED
}
ACTION_TO_EVENT = {EXECUTE: executor.EXECUTED, REVERT: executor.REVERTED}
# NOTE(skudriashev): A timeout which specifies request expiration period.
REQUEST_TIMEOUT = 60
@@ -149,9 +146,11 @@ class Message(metaclass=abc.ABCMeta):
"""Base class for all message types."""
def __repr__(self):
return ("<%s object at 0x%x with contents %s>"
% (reflection.get_class_name(self, fully_qualified=False),
id(self), self.to_dict()))
return "<%s object at 0x%x with contents %s>" % (
reflection.get_class_name(self, fully_qualified=False),
id(self),
self.to_dict(),
)
@abc.abstractmethod
def to_dict(self):
@@ -180,7 +179,7 @@ class Notify(Message):
"items": {
"type": "string",
},
}
},
},
"required": ["topic", 'tasks'],
"additionalProperties": False,
@@ -217,21 +216,24 @@ class Notify(Message):
except su.ValidationError as e:
cls_name = reflection.get_class_name(cls, fully_qualified=False)
if response:
excp.raise_with_cause(excp.InvalidFormat,
"%s message response data not of the"
" expected format: %s" % (cls_name,
e.message),
cause=e)
excp.raise_with_cause(
excp.InvalidFormat,
"%s message response data not of the"
" expected format: %s" % (cls_name, e.message),
cause=e,
)
else:
excp.raise_with_cause(excp.InvalidFormat,
"%s message sender data not of the"
" expected format: %s" % (cls_name,
e.message),
cause=e)
excp.raise_with_cause(
excp.InvalidFormat,
"%s message sender data not of the"
" expected format: %s" % (cls_name, e.message),
cause=e,
)
_WorkUnit = collections.namedtuple('_WorkUnit', ['task_cls', 'task_name',
'action', 'arguments'])
_WorkUnit = collections.namedtuple(
'_WorkUnit', ['task_cls', 'task_name', 'action', 'arguments']
)
class Request(Message):
@@ -299,9 +301,16 @@ class Request(Message):
'required': ['task_cls', 'task_name', 'task_version', 'action'],
}
def __init__(self, task, uuid, action,
arguments, timeout=REQUEST_TIMEOUT, result=NO_RESULT,
failures=None):
def __init__(
self,
task,
uuid,
action,
arguments,
timeout=REQUEST_TIMEOUT,
result=NO_RESULT,
failures=None,
):
self._action = action
self._event = ACTION_TO_EVENT[action]
self._arguments = arguments
@@ -383,8 +392,12 @@ class Request(Message):
try:
moved = self.transition(new_state)
except excp.InvalidState:
logger.warnng("Failed to transition '%s' to %s state.", self,
new_state, exc_info=True)
logger.warnng(
"Failed to transition '%s' to %s state.",
self,
new_state,
exc_info=True,
)
return moved
@fasteners.locked
@@ -402,14 +415,19 @@ class Request(Message):
try:
self._machine.process_event(make_an_event(new_state))
except (machine_excp.NotFound, machine_excp.InvalidState) as e:
raise excp.InvalidState("Request transition from %s to %s is"
" not allowed: %s" % (old_state,
new_state, e))
raise excp.InvalidState(
"Request transition from %s to %s is"
" not allowed: %s" % (old_state, new_state, e)
)
else:
if new_state in STOP_TIMER_STATES:
self._watch.stop()
LOG.debug("Transitioned '%s' from %s state to %s state", self,
old_state, new_state)
LOG.debug(
"Transitioned '%s' from %s state to %s state",
self,
old_state,
new_state,
)
return True
@classmethod
@@ -418,11 +436,12 @@ class Request(Message):
su.schema_validate(data, cls.SCHEMA)
except su.ValidationError as e:
cls_name = reflection.get_class_name(cls, fully_qualified=False)
excp.raise_with_cause(excp.InvalidFormat,
"%s message response data not of the"
" expected format: %s" % (cls_name,
e.message),
cause=e)
excp.raise_with_cause(
excp.InvalidFormat,
"%s message response data not of the"
" expected format: %s" % (cls_name, e.message),
cause=e,
)
else:
# Validate all failure dictionaries that *may* be present...
failures = []
@@ -556,11 +575,12 @@ class Response(Message):
su.schema_validate(data, cls.SCHEMA)
except su.ValidationError as e:
cls_name = reflection.get_class_name(cls, fully_qualified=False)
excp.raise_with_cause(excp.InvalidFormat,
"%s message response data not of the"
" expected format: %s" % (cls_name,
e.message),
cause=e)
excp.raise_with_cause(
excp.InvalidFormat,
"%s message response data not of the"
" expected format: %s" % (cls_name, e.message),
cause=e,
)
else:
state = data['state']
if state == FAILURE and 'result' in data:

102
taskflow/engines/worker_based/proxy.py
View File

@@ -31,11 +31,13 @@ DRAIN_EVENTS_PERIOD = 1
# instead of returning the raw results from the kombu connection objects
# themselves so that a person can not mutate those objects (which would be
# bad).
_ConnectionDetails = collections.namedtuple('_ConnectionDetails',
['uri', 'transport'])
_TransportDetails = collections.namedtuple('_TransportDetails',
['options', 'driver_type',
'driver_name', 'driver_version'])
_ConnectionDetails = collections.namedtuple(
'_ConnectionDetails', ['uri', 'transport']
)
_TransportDetails = collections.namedtuple(
'_TransportDetails',
['options', 'driver_type', 'driver_name', 'driver_version'],
)
class Proxy:
@@ -65,10 +67,17 @@ class Proxy:
# value is valid...
_RETRY_INT_OPTS = frozenset(['max_retries'])
def __init__(self, topic, exchange,
type_handlers=None, on_wait=None, url=None,
transport=None, transport_options=None,
retry_options=None):
def __init__(
self,
topic,
exchange,
type_handlers=None,
on_wait=None,
url=None,
transport=None,
transport_options=None,
retry_options=None,
):
self._topic = topic
self._exchange_name = exchange
self._on_wait = on_wait
@@ -77,7 +86,8 @@ class Proxy:
# NOTE(skudriashev): Process all incoming messages only if proxy is
# running, otherwise requeue them.
requeue_filters=[lambda data, message: not self.is_running],
type_handlers=type_handlers)
type_handlers=type_handlers,
)
ensure_options = self.DEFAULT_RETRY_OPTIONS.copy()
if retry_options is not None:
@@ -91,9 +101,11 @@ class Proxy:
else:
tmp_val = float(val)
if tmp_val < 0:
raise ValueError("Expected value greater or equal to"
" zero for 'retry_options' %s; got"
" %s instead" % (k, val))
raise ValueError(
"Expected value greater or equal to"
" zero for 'retry_options' %s; got"
" %s instead" % (k, val)
)
ensure_options[k] = tmp_val
self._ensure_options = ensure_options
@@ -104,12 +116,14 @@ class Proxy:
self._drain_events_timeout = polling_interval
# create connection
self._conn = kombu.Connection(url, transport=transport,
transport_options=transport_options)
self._conn = kombu.Connection(
url, transport=transport, transport_options=transport_options
)
# create exchange
self._exchange = kombu.Exchange(name=self._exchange_name,
durable=False, auto_delete=True)
self._exchange = kombu.Exchange(
name=self._exchange_name, durable=False, auto_delete=True
)
@property
def dispatcher(self):
@@ -131,10 +145,11 @@ class Proxy:
options=transport_options,
driver_type=self._conn.transport.driver_type,
driver_name=self._conn.transport.driver_name,
driver_version=driver_version)
driver_version=driver_version,
)
return _ConnectionDetails(
uri=self._conn.as_uri(include_password=False),
transport=transport)
uri=self._conn.as_uri(include_password=False), transport=transport
)
@property
def is_running(self):
@@ -144,10 +159,14 @@ class Proxy:
def _make_queue(self, routing_key, exchange, channel=None):
"""Make a named queue for the given exchange."""
queue_name = f"{self._exchange_name}_{routing_key}"
return kombu.Queue(name=queue_name,
routing_key=routing_key, durable=False,
exchange=exchange, auto_delete=True,
channel=channel)
return kombu.Queue(
name=queue_name,
routing_key=routing_key,
durable=False,
exchange=exchange,
auto_delete=True,
channel=channel,
)
def publish(self, msg, routing_key, reply_to=None, correlation_id=None):
"""Publish message to the named exchange with given routing key."""
@@ -159,27 +178,33 @@ class Proxy:
# Filter out any empty keys...
routing_keys = [r_k for r_k in routing_keys if r_k]
if not routing_keys:
LOG.warning("No routing key/s specified; unable to send '%s'"
" to any target queue on exchange '%s'", msg,
self._exchange_name)
LOG.warning(
"No routing key/s specified; unable to send '%s'"
" to any target queue on exchange '%s'",
msg,
self._exchange_name,
)
return
def _publish(producer, routing_key):
queue = self._make_queue(routing_key, self._exchange)
producer.publish(body=msg.to_dict(),
routing_key=routing_key,
exchange=self._exchange,
declare=[queue],
type=msg.TYPE,
reply_to=reply_to,
correlation_id=correlation_id)
producer.publish(
body=msg.to_dict(),
routing_key=routing_key,
exchange=self._exchange,
declare=[queue],
type=msg.TYPE,
reply_to=reply_to,
correlation_id=correlation_id,
)
def _publish_errback(exc, interval):
LOG.exception('Publishing error: %s', exc)
LOG.info('Retry triggering in %s seconds', interval)
LOG.debug("Sending '%s' message using routing keys %s",
msg, routing_keys)
LOG.debug(
"Sending '%s' message using routing keys %s", msg, routing_keys
)
with kombu.connections[self._conn].acquire(block=True) as conn:
with conn.Producer() as producer:
ensure_kwargs = self._ensure_options.copy()
@@ -201,8 +226,9 @@ class Proxy:
LOG.exception('Draining error: %s', exc)
LOG.info('Retry triggering in %s seconds', interval)
LOG.info("Starting to consume from the '%s' exchange.",
self._exchange_name)
LOG.info(
"Starting to consume from the '%s' exchange.", self._exchange_name
)
with kombu.connections[self._conn].acquire(block=True) as conn:
queue = self._make_queue(self._topic, self._exchange, channel=conn)
callbacks = [self._dispatcher.on_message]

198
taskflow/engines/worker_based/server.py
View File

@@ -32,27 +32,41 @@ LOG = logging.getLogger(__name__)
class Server:
"""Server implementation that waits for incoming tasks requests."""
def __init__(self, topic, exchange, executor, endpoints,
url=None, transport=None, transport_options=None,
retry_options=None):
def __init__(
self,
topic,
exchange,
executor,
endpoints,
url=None,
transport=None,
transport_options=None,
retry_options=None,
):
type_handlers = {
pr.NOTIFY: dispatcher.Handler(
self._delayed_process(self._process_notify),
validator=functools.partial(pr.Notify.validate,
response=False)),
validator=functools.partial(
pr.Notify.validate, response=False
),
),
pr.REQUEST: dispatcher.Handler(
self._delayed_process(self._process_request),
validator=pr.Request.validate),
validator=pr.Request.validate,
),
}
self._executor = executor
self._proxy = proxy.Proxy(topic, exchange,
type_handlers=type_handlers,
url=url, transport=transport,
transport_options=transport_options,
retry_options=retry_options)
self._proxy = proxy.Proxy(
topic,
exchange,
type_handlers=type_handlers,
url=url,
transport=transport,
transport_options=transport_options,
retry_options=retry_options,
)
self._topic = topic
self._endpoints = {endpoint.name: endpoint
for endpoint in endpoints}
self._endpoints = {endpoint.name: endpoint for endpoint in endpoints}
def _delayed_process(self, func):
"""Runs the function using the instances executor (eventually).
@@ -65,25 +79,34 @@ class Server:
func_name = reflection.get_callable_name(func)
def _on_run(watch, content, message):
LOG.trace("It took %s seconds to get around to running"
" function/method '%s' with"
" message '%s'", watch.elapsed(), func_name,
ku.DelayedPretty(message))
LOG.trace(
"It took %s seconds to get around to running"
" function/method '%s' with"
" message '%s'",
watch.elapsed(),
func_name,
ku.DelayedPretty(message),
)
return func(content, message)
def _on_receive(content, message):
LOG.debug("Submitting message '%s' for execution in the"
" future to '%s'", ku.DelayedPretty(message), func_name)
LOG.debug(
"Submitting message '%s' for execution in the future to '%s'",
ku.DelayedPretty(message),
func_name,
)
watch = timeutils.StopWatch()
watch.start()
try:
self._executor.submit(_on_run, watch, content, message)
except RuntimeError:
LOG.error("Unable to continue processing message '%s',"
" submission to instance executor (with later"
" execution by '%s') was unsuccessful",
ku.DelayedPretty(message), func_name,
exc_info=True)
LOG.exception(
"Unable to continue processing message '%s',"
" submission to instance executor (with later"
" execution by '%s') was unsuccessful",
ku.DelayedPretty(message),
func_name,
)
return _on_receive
@@ -103,8 +126,7 @@ class Server:
try:
properties.append(message.properties[prop])
except KeyError:
raise ValueError("The '%s' message property is missing" %
prop)
raise ValueError("The '%s' message property is missing" % prop)
return properties
def _reply(self, capture, reply_to, task_uuid, state=pr.FAILURE, **kwargs):
@@ -122,9 +144,13 @@ class Server:
except Exception:
if not capture:
raise
LOG.critical("Failed to send reply to '%s' for task '%s' with"
" response %s", reply_to, task_uuid, response,
exc_info=True)
LOG.critical(
"Failed to send reply to '%s' for task '%s' with response %s",
reply_to,
task_uuid,
response,
exc_info=True,
)
return published
def _on_event(self, reply_to, task_uuid, event_type, details):
@@ -132,26 +158,39 @@ class Server:
# NOTE(harlowja): the executor that will trigger this using the
# task notification/listener mechanism will handle logging if this
# fails, so thats why capture is 'False' is used here.
self._reply(False, reply_to, task_uuid, pr.EVENT,
event_type=event_type, details=details)
self._reply(
False,
reply_to,
task_uuid,
pr.EVENT,
event_type=event_type,
details=details,
)
def _process_notify(self, notify, message):
"""Process notify message and reply back."""
try:
reply_to = message.properties['reply_to']
except KeyError:
LOG.warning("The 'reply_to' message property is missing"
" in received notify message '%s'",
ku.DelayedPretty(message), exc_info=True)
LOG.warning(
"The 'reply_to' message property is missing"
" in received notify message '%s'",
ku.DelayedPretty(message),
exc_info=True,
)
else:
response = pr.Notify(topic=self._topic,
tasks=list(self._endpoints.keys()))
response = pr.Notify(
topic=self._topic, tasks=list(self._endpoints.keys())
)
try:
self._proxy.publish(response, routing_key=reply_to)
except Exception:
LOG.critical("Failed to send reply to '%s' with notify"
" response '%s'", reply_to, response,
exc_info=True)
LOG.critical(
"Failed to send reply to '%s' with notify response '%s'",
reply_to,
response,
exc_info=True,
)
def _process_request(self, request, message):
"""Process request message and reply back."""
@@ -162,22 +201,28 @@ class Server:
# in the first place...).
reply_to, task_uuid = self._parse_message(message)
except ValueError:
LOG.warning("Failed to parse request attributes from message '%s'",
ku.DelayedPretty(message), exc_info=True)
LOG.warning(
"Failed to parse request attributes from message '%s'",
ku.DelayedPretty(message),
exc_info=True,
)
return
else:
# prepare reply callback
reply_callback = functools.partial(self._reply, True, reply_to,
task_uuid)
reply_callback = functools.partial(
self._reply, True, reply_to, task_uuid
)
# Parse the request to get the activity/work to perform.
try:
work = pr.Request.from_dict(request, task_uuid=task_uuid)
except ValueError:
with misc.capture_failure() as failure:
LOG.warning("Failed to parse request contents"
" from message '%s'",
ku.DelayedPretty(message), exc_info=True)
LOG.warning(
"Failed to parse request contents from message '%s'",
ku.DelayedPretty(message),
exc_info=True,
)
reply_callback(result=pr.failure_to_dict(failure))
return
@@ -186,10 +231,13 @@ class Server:
endpoint = self._endpoints[work.task_cls]
except KeyError:
with misc.capture_failure() as failure:
LOG.warning("The '%s' task endpoint does not exist, unable"
" to continue processing request message '%s'",
work.task_cls, ku.DelayedPretty(message),
exc_info=True)
LOG.warning(
"The '%s' task endpoint does not exist, unable"
" to continue processing request message '%s'",
work.task_cls,
ku.DelayedPretty(message),
exc_info=True,
)
reply_callback(result=pr.failure_to_dict(failure))
return
else:
@@ -197,10 +245,15 @@ class Server:
handler = getattr(endpoint, work.action)
except AttributeError:
with misc.capture_failure() as failure:
LOG.warning("The '%s' handler does not exist on task"
" endpoint '%s', unable to continue processing"
" request message '%s'", work.action, endpoint,
ku.DelayedPretty(message), exc_info=True)
LOG.warning(
"The '%s' handler does not exist on task"
" endpoint '%s', unable to continue processing"
" request message '%s'",
work.action,
endpoint,
ku.DelayedPretty(message),
exc_info=True,
)
reply_callback(result=pr.failure_to_dict(failure))
return
else:
@@ -208,10 +261,14 @@ class Server:
task = endpoint.generate(name=work.task_name)
except Exception:
with misc.capture_failure() as failure:
LOG.warning("The '%s' task '%s' generation for request"
" message '%s' failed", endpoint,
work.action, ku.DelayedPretty(message),
exc_info=True)
LOG.warning(
"The '%s' task '%s' generation for request"
" message '%s' failed",
endpoint,
work.action,
ku.DelayedPretty(message),
exc_info=True,
)
reply_callback(result=pr.failure_to_dict(failure))
return
else:
@@ -222,24 +279,31 @@ class Server:
# emit them back to the engine... for handling at the engine side
# of things...
if task.notifier.can_be_registered(nt.Notifier.ANY):
task.notifier.register(nt.Notifier.ANY,
functools.partial(self._on_event,
reply_to, task_uuid))
task.notifier.register(
nt.Notifier.ANY,
functools.partial(self._on_event, reply_to, task_uuid),
)
elif isinstance(task.notifier, nt.RestrictedNotifier):
# Only proxy the allowable events then...
for event_type in task.notifier.events_iter():
task.notifier.register(event_type,
functools.partial(self._on_event,
reply_to, task_uuid))
task.notifier.register(
event_type,
functools.partial(self._on_event, reply_to, task_uuid),
)
# Perform the task action.
try:
result = handler(task, **work.arguments)
except Exception:
with misc.capture_failure() as failure:
LOG.warning("The '%s' endpoint '%s' execution for request"
" message '%s' failed", endpoint, work.action,
ku.DelayedPretty(message), exc_info=True)
LOG.warning(
"The '%s' endpoint '%s' execution for request"
" message '%s' failed",
endpoint,
work.action,
ku.DelayedPretty(message),
exc_info=True,
)
reply_callback(result=pr.failure_to_dict(failure))
else:
# And be done with it!

58
taskflow/engines/worker_based/types.py
View File

@@ -71,19 +71,26 @@ class TopicWorker:
r = reflection.get_class_name(self, fully_qualified=False)
if self.identity is not self._NO_IDENTITY:
r += "(identity={}, tasks={}, topic={})".format(
self.identity, self.tasks, self.topic)
self.identity, self.tasks, self.topic
)
else:
r += "(identity=*, tasks={}, topic={})".format(
self.tasks, self.topic)
self.tasks, self.topic
)
return r
class ProxyWorkerFinder:
"""Requests and receives responses about workers topic+task details."""
def __init__(self, uuid, proxy, topics,
beat_periodicity=pr.NOTIFY_PERIOD,
worker_expiry=pr.EXPIRES_AFTER):
def __init__(
self,
uuid,
proxy,
topics,
beat_periodicity=pr.NOTIFY_PERIOD,
worker_expiry=pr.EXPIRES_AFTER,
):
self._cond = threading.Condition()
self._proxy = proxy
self._topics = topics
@@ -134,7 +141,7 @@ class ProxyWorkerFinder:
if len(available_workers) == 1:
return available_workers[0]
else:
return random.choice(available_workers)
return random.choice(available_workers) # noqa: S311
@property
def messages_processed(self):
@@ -157,14 +164,14 @@ class ProxyWorkerFinder:
match workers to tasks to run).
"""
if self._messages_published == 0:
self._proxy.publish(pr.Notify(),
self._topics, reply_to=self._uuid)
self._proxy.publish(pr.Notify(), self._topics, reply_to=self._uuid)
self._messages_published += 1
self._watch.restart()
else:
if self._watch.expired():
self._proxy.publish(pr.Notify(),
self._topics, reply_to=self._uuid)
self._proxy.publish(
pr.Notify(), self._topics, reply_to=self._uuid
)
self._messages_published += 1
self._watch.restart()
@@ -188,16 +195,21 @@ class ProxyWorkerFinder:
def process_response(self, data, message):
"""Process notify message sent from remote side."""
LOG.debug("Started processing notify response message '%s'",
ku.DelayedPretty(message))
LOG.debug(
"Started processing notify response message '%s'",
ku.DelayedPretty(message),
)
response = pr.Notify(**data)
LOG.debug("Extracted notify response '%s'", response)
with self._cond:
worker, new_or_updated = self._add(response.topic,
response.tasks)
worker, new_or_updated = self._add(response.topic, response.tasks)
if new_or_updated:
LOG.debug("Updated worker '%s' (%s total workers are"
" currently known)", worker, self.total_workers)
LOG.debug(
"Updated worker '%s' (%s total workers are"
" currently known)",
worker,
self.total_workers,
)
self._cond.notify_all()
worker.last_seen = timeutils.now()
self._messages_processed += 1
@@ -207,8 +219,9 @@ class ProxyWorkerFinder:
Returns how many workers were removed.
"""
if (not self._workers or
(self._worker_expiry is None or self._worker_expiry <= 0)):
if not self._workers or (
self._worker_expiry is None or self._worker_expiry <= 0
):
return 0
dead_workers = {}
with self._cond:
@@ -225,9 +238,12 @@ class ProxyWorkerFinder:
self._cond.notify_all()
if dead_workers and LOG.isEnabledFor(logging.INFO):
for worker, secs_since_last_seen in dead_workers.values():
LOG.info("Removed worker '%s' as it has not responded to"
" notification requests in %0.3f seconds",
worker, secs_since_last_seen)
LOG.info(
"Removed worker '%s' as it has not responded to"
" notification requests in %0.3f seconds",
worker,
secs_since_last_seen,
)
return len(dead_workers)
def reset(self):

43
taskflow/engines/worker_based/worker.py
View File

@@ -55,24 +55,38 @@ class Worker:
(see: :py:attr:`~.proxy.Proxy.DEFAULT_RETRY_OPTIONS`)
"""
def __init__(self, exchange, topic, tasks,
executor=None, threads_count=None, url=None,
transport=None, transport_options=None,
retry_options=None):
def __init__(
self,
exchange,
topic,
tasks,
executor=None,
threads_count=None,
url=None,
transport=None,
transport_options=None,
retry_options=None,
):
self._topic = topic
self._executor = executor
self._owns_executor = False
if self._executor is None:
self._executor = futurist.ThreadPoolExecutor(
max_workers=threads_count)
max_workers=threads_count
)
self._owns_executor = True
self._endpoints = self._derive_endpoints(tasks)
self._exchange = exchange
self._server = server.Server(topic, exchange, self._executor,
self._endpoints, url=url,
transport=transport,
transport_options=transport_options,
retry_options=retry_options)
self._server = server.Server(
topic,
exchange,
self._executor,
self._endpoints,
url=url,
transport=transport,
transport_options=transport_options,
retry_options=retry_options,
)
@staticmethod
def _derive_endpoints(tasks):
@@ -86,8 +100,9 @@ class Worker:
connection_details = self._server.connection_details
transport = connection_details.transport
if transport.driver_version:
transport_driver = "{} v{}".format(transport.driver_name,
transport.driver_version)
transport_driver = "{} v{}".format(
transport.driver_name, transport.driver_version
)
else:
transport_driver = transport.driver_name
try:
@@ -145,12 +160,12 @@ class Worker:
if __name__ == '__main__':
import argparse
import logging as log
parser = argparse.ArgumentParser()
parser.add_argument("--exchange", required=True)
parser.add_argument("--connection-url", required=True)
parser.add_argument("--topic", required=True)
parser.add_argument("--task", action='append',
metavar="TASK", default=[])
parser.add_argument("--task", action='append', metavar="TASK", default=[])
parser.add_argument("-v", "--verbose", action='store_true')
args = parser.parse_args()
if args.verbose:

59
taskflow/examples/99_bottles.py
View File

@@ -22,9 +22,9 @@ import traceback
from kazoo import client
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow.conductors import backends as conductor_backends
@@ -90,16 +90,19 @@ def make_bottles(count):
s = lf.Flow("bottle-song")
take_bottle = TakeABottleDown("take-bottle-%s" % count,
inject={'bottles_left': count},
provides='bottles_left')
take_bottle = TakeABottleDown(
"take-bottle-%s" % count,
inject={'bottles_left': count},
provides='bottles_left',
)
pass_it = PassItAround("pass-%s-around" % count)
next_bottles = Conclusion("next-bottles-%s" % (count - 1))
s.add(take_bottle, pass_it, next_bottles)
for bottle in reversed(list(range(1, count))):
take_bottle = TakeABottleDown("take-bottle-%s" % bottle,
provides='bottles_left')
take_bottle = TakeABottleDown(
"take-bottle-%s" % bottle, provides='bottles_left'
)
pass_it = PassItAround("pass-%s-around" % bottle)
next_bottles = Conclusion("next-bottles-%s" % (bottle - 1))
s.add(take_bottle, pass_it, next_bottles)
@@ -122,15 +125,17 @@ def run_conductor(only_run_once=False):
if event.endswith("_start"):
w = timeutils.StopWatch()
w.start()
base_event = event[0:-len("_start")]
base_event = event[0 : -len("_start")]
event_watches[base_event] = w
if event.endswith("_end"):
base_event = event[0:-len("_end")]
base_event = event[0 : -len("_end")]
try:
w = event_watches.pop(base_event)
w.stop()
print("It took %0.3f seconds for event '%s' to finish"
% (w.elapsed(), base_event))
print(
"It took %0.3f seconds for event '%s' to finish"
% (w.elapsed(), base_event)
)
except KeyError:
pass
if event == 'running_end' and only_run_once:
@@ -142,12 +147,14 @@ def run_conductor(only_run_once=False):
with contextlib.closing(persist_backend):
with contextlib.closing(persist_backend.get_connection()) as conn:
conn.upgrade()
job_backend = job_backends.fetch(my_name, JB_CONF,
persistence=persist_backend)
job_backend = job_backends.fetch(
my_name, JB_CONF, persistence=persist_backend
)
job_backend.connect()
with contextlib.closing(job_backend):
cond = conductor_backends.fetch('blocking', my_name, job_backend,
persistence=persist_backend)
cond = conductor_backends.fetch(
'blocking', my_name, job_backend, persistence=persist_backend
)
on_conductor_event = functools.partial(on_conductor_event, cond)
cond.notifier.register(cond.notifier.ANY, on_conductor_event)
# Run forever, and kill -9 or ctrl-c me...
@@ -166,8 +173,9 @@ def run_poster():
with contextlib.closing(persist_backend):
with contextlib.closing(persist_backend.get_connection()) as conn:
conn.upgrade()
job_backend = job_backends.fetch(my_name, JB_CONF,
persistence=persist_backend)
job_backend = job_backends.fetch(
my_name, JB_CONF, persistence=persist_backend
)
job_backend.connect()
with contextlib.closing(job_backend):
# Create information in the persistence backend about the
@@ -175,14 +183,19 @@ def run_poster():
# can be called to create the tasks that the work unit needs
# to be done.
lb = models.LogBook("post-from-%s" % my_name)
fd = models.FlowDetail("song-from-%s" % my_name,
uuidutils.generate_uuid())
fd = models.FlowDetail(
"song-from-%s" % my_name, uuidutils.generate_uuid()
)
lb.add(fd)
with contextlib.closing(persist_backend.get_connection()) as conn:
conn.save_logbook(lb)
engines.save_factory_details(fd, make_bottles,
[HOW_MANY_BOTTLES], {},
backend=persist_backend)
engines.save_factory_details(
fd,
make_bottles,
[HOW_MANY_BOTTLES],
{},
backend=persist_backend,
)
# Post, and be done with it!
jb = job_backend.post("song-from-%s" % my_name, book=lb)
print("Posted: %s" % jb)

11
taskflow/examples/alphabet_soup.py
View File

@@ -23,9 +23,9 @@ import time
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)
@@ -74,8 +74,9 @@ print("Constructing...")
soup = linear_flow.Flow("alphabet-soup")
for letter in string.ascii_lowercase:
abc = AlphabetTask(letter)
abc.notifier.register(task.EVENT_UPDATE_PROGRESS,
functools.partial(progress_printer, abc))
abc.notifier.register(
task.EVENT_UPDATE_PROGRESS, functools.partial(progress_printer, abc)
)
soup.add(abc)
try:
print("Loading...")

32
taskflow/examples/build_a_car.py
View File

@@ -19,9 +19,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
@@ -64,6 +64,7 @@ def build_wheels():
# These just return true to indiciate success, they would in the real work
# do more than just that.
def install_engine(frame, engine):
return True
@@ -130,15 +131,22 @@ flow = lf.Flow("make-auto").add(
task.FunctorTask(install_engine, provides='engine_installed'),
task.FunctorTask(install_doors, provides='doors_installed'),
task.FunctorTask(install_windows, provides='windows_installed'),
task.FunctorTask(install_wheels, provides='wheels_installed')),
task.FunctorTask(verify, requires=['frame',
'engine',
'doors',
'wheels',
'engine_installed',
'doors_installed',
'windows_installed',
'wheels_installed']))
task.FunctorTask(install_wheels, provides='wheels_installed'),
),
task.FunctorTask(
verify,
requires=[
'frame',
'engine',
'doors',
'wheels',
'engine_installed',
'doors_installed',
'windows_installed',
'wheels_installed',
],
),
)
# This dictionary will be provided to the tasks as a specification for what
# the tasks should produce, in this example this specification will influence

29
taskflow/examples/buildsystem.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import taskflow.engines
@@ -40,17 +40,18 @@ import example_utils as eu # noqa
class CompileTask(task.Task):
"""Pretends to take a source and make object file."""
default_provides = 'object_filename'
def execute(self, source_filename):
object_filename = '%s.o' % os.path.splitext(source_filename)[0]
print('Compiling %s into %s'
% (source_filename, object_filename))
print('Compiling %s into %s' % (source_filename, object_filename))
return object_filename
class LinkTask(task.Task):
"""Pretends to link executable form several object files."""
default_provides = 'executable'
def __init__(self, executable_path, *args, **kwargs):
@@ -59,14 +60,16 @@ class LinkTask(task.Task):
def execute(self, **kwargs):
object_filenames = list(kwargs.values())
print('Linking executable %s from files %s'
% (self._executable_path,
', '.join(object_filenames)))
print(
'Linking executable %s from files %s'
% (self._executable_path, ', '.join(object_filenames))
)
return self._executable_path
class BuildDocsTask(task.Task):
"""Pretends to build docs from sources."""
default_provides = 'docs'
def execute(self, **kwargs):
@@ -84,9 +87,13 @@ def make_flow_and_store(source_files, executable_only=False):
object_stored = '%s-object' % source
store[source_stored] = source
object_targets.append(object_stored)
flow.add(CompileTask(name='compile-%s' % source,
rebind={'source_filename': source_stored},
provides=object_stored))
flow.add(
CompileTask(
name='compile-%s' % source,
rebind={'source_filename': source_stored},
provides=object_stored,
)
)
flow.add(BuildDocsTask(requires=list(store.keys())))
# Try this to see executable_only switch broken:

12
taskflow/examples/calculate_in_parallel.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import taskflow.engines
@@ -65,8 +65,7 @@ flow = lf.Flow('root').add(
# Provide the initial values for other tasks to depend on.
#
# x1 = 2, y1 = 3, x2 = 5, x3 = 8
Provider("provide-adder", 2, 3, 5, 8,
provides=('x1', 'y1', 'x2', 'y2')),
Provider("provide-adder", 2, 3, 5, 8, provides=('x1', 'y1', 'x2', 'y2')),
# Note here that we define the flow that contains the 2 adders to be an
# unordered flow since the order in which these execute does not matter,
# another way to solve this would be to use a graph_flow pattern, which
@@ -85,7 +84,8 @@ flow = lf.Flow('root').add(
Adder(name="add-2", provides='z2', rebind=['x2', 'y2']),
),
# r = z1+z2 = 18
Adder(name="sum-1", provides='r', rebind=['z1', 'z2']))
Adder(name="sum-1", provides='r', rebind=['z1', 'z2']),
)
# The result here will be all results (from all tasks) which is stored in an

12
taskflow/examples/calculate_linear.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import taskflow.engines
@@ -50,7 +50,6 @@ from taskflow import task
# storage backend before your tasks are ran (which accomplishes a similar goal
# in a more uniform manner).
class Provider(task.Task):
def __init__(self, name, *args, **kwargs):
super().__init__(name=name, **kwargs)
self._provide = args
@@ -77,8 +76,7 @@ class Adder(task.Task):
# this function needs to undo if some later operation fails.
class Multiplier(task.Task):
def __init__(self, name, multiplier, provides=None, rebind=None):
super().__init__(name=name, provides=provides,
rebind=rebind)
super().__init__(name=name, provides=provides, rebind=rebind)
self._multiplier = multiplier
def execute(self, z):
@@ -104,7 +102,7 @@ flow = lf.Flow('root').add(
# bound to the 'z' variable provided from the above 'provider' object but
# instead the 'z' argument will be taken from the 'a' variable provided
# by the second add-2 listed above.
Multiplier("multi", 3, provides='r', rebind={'z': 'a'})
Multiplier("multi", 3, provides='r', rebind={'z': 'a'}),
)
# The result here will be all results (from all tasks) which is stored in an

6
taskflow/examples/create_parallel_volume.py
View File

@@ -21,9 +21,9 @@ import time
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from oslo_utils import reflection

9
taskflow/examples/delayed_return.py
View File

@@ -21,9 +21,9 @@ from concurrent import futures
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)
@@ -45,7 +45,8 @@ class PokeFutureListener(base.Listener):
super().__init__(
engine,
task_listen_for=(notifier.Notifier.ANY,),
flow_listen_for=[])
flow_listen_for=[],
)
self._future = future
self._task_name = task_name

57
taskflow/examples/distance_calculator.py
View File

@@ -17,9 +17,9 @@ import math
import os
import sys
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow import engines
@@ -60,24 +60,39 @@ if __name__ == '__main__':
any_distance = linear_flow.Flow("origin").add(DistanceTask())
results = engines.run(any_distance)
print(results)
print("{} is near-enough to {}: {}".format(
results['distance'], 0.0, is_near(results['distance'], 0.0)))
print(
"{} is near-enough to {}: {}".format(
results['distance'], 0.0, is_near(results['distance'], 0.0)
)
)
results = engines.run(any_distance, store={'a': Point(1, 1)})
print(results)
print("{} is near-enough to {}: {}".format(
results['distance'], 1.4142, is_near(results['distance'], 1.4142)))
print(
"{} is near-enough to {}: {}".format(
results['distance'], 1.4142, is_near(results['distance'], 1.4142)
)
)
results = engines.run(any_distance, store={'a': Point(10, 10)})
print(results)
print("{} is near-enough to {}: {}".format(
results['distance'], 14.14199, is_near(results['distance'], 14.14199)))
print(
"{} is near-enough to {}: {}".format(
results['distance'],
14.14199,
is_near(results['distance'], 14.14199),
)
)
results = engines.run(any_distance,
store={'a': Point(5, 5), 'b': Point(10, 10)})
results = engines.run(
any_distance, store={'a': Point(5, 5), 'b': Point(10, 10)}
)
print(results)
print("{} is near-enough to {}: {}".format(
results['distance'], 7.07106, is_near(results['distance'], 7.07106)))
print(
"{} is near-enough to {}: {}".format(
results['distance'], 7.07106, is_near(results['distance'], 7.07106)
)
)
# For this we use the ability to override at task creation time the
# optional arguments so that we don't need to continue to send them
@@ -88,10 +103,18 @@ if __name__ == '__main__':
ten_distance.add(DistanceTask(inject={'a': Point(10, 10)}))
results = engines.run(ten_distance, store={'b': Point(10, 10)})
print(results)
print("{} is near-enough to {}: {}".format(
results['distance'], 0.0, is_near(results['distance'], 0.0)))
print(
"{} is near-enough to {}: {}".format(
results['distance'], 0.0, is_near(results['distance'], 0.0)
)
)
results = engines.run(ten_distance)
print(results)
print("{} is near-enough to {}: {}".format(
results['distance'], 14.14199, is_near(results['distance'], 14.14199)))
print(
"{} is near-enough to {}: {}".format(
results['distance'],
14.14199,
is_near(results['distance'], 14.14199),
)
)

7
taskflow/examples/dump_memory_backend.py
View File

@@ -19,9 +19,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)
@@ -39,6 +39,7 @@ class PrintTask(task.Task):
def execute(self):
print("Running '%s'" % self.name)
# Make a little flow and run it...
f = lf.Flow('root')
for alpha in ['a', 'b', 'c']:

6
taskflow/examples/echo_listener.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.DEBUG)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow import engines

8
taskflow/examples/example_utils.py
View File

@@ -28,6 +28,7 @@ LOG = logging.getLogger(__name__)
try:
import sqlalchemy as _sa # noqa
SQLALCHEMY_AVAILABLE = True
except ImportError:
SQLALCHEMY_AVAILABLE = False
@@ -93,8 +94,11 @@ def get_backend(backend_uri=None):
if not tmp_dir:
tmp_dir = tempfile.mkdtemp()
backend_uri = "file:///%s" % tmp_dir
LOG.exception("Falling back to file backend using temporary"
" directory located at: %s", tmp_dir)
LOG.exception(
"Falling back to file backend using temporary"
" directory located at: %s",
tmp_dir,
)
backend = backends.fetch(_make_conf(backend_uri))
else:
raise e

13
taskflow/examples/fake_billing.py
View File

@@ -20,9 +20,9 @@ import time
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from oslo_utils import uuidutils
@@ -130,8 +130,11 @@ class ActivateDriver(task.Task):
# that the url sending helper class uses. This allows the task progress
# to be tied to the url sending progress, which is very useful for
# downstream systems to be aware of what a task is doing at any time.
url_sender.send(self._url, json.dumps(parsed_request),
status_cb=self.update_progress)
url_sender.send(
self._url,
json.dumps(parsed_request),
status_cb=self.update_progress,
)
return self._url
def update_progress(self, progress, **kwargs):

22
taskflow/examples/graph_flow.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import taskflow.engines
@@ -46,7 +46,6 @@ from taskflow import task
class Adder(task.Task):
def execute(self, x, y):
return x + y
@@ -56,7 +55,7 @@ flow = gf.Flow('root').add(
# x2 = y3+y4 = 12
Adder("add2", provides='x2', rebind=['y3', 'y4']),
# x1 = y1+y2 = 4
Adder("add1", provides='x1', rebind=['y1', 'y2'])
Adder("add1", provides='x1', rebind=['y1', 'y2']),
),
# x5 = x1+x3 = 20
Adder("add5", provides='x5', rebind=['x1', 'x3']),
@@ -67,7 +66,8 @@ flow = gf.Flow('root').add(
# x6 = x5+x4 = 41
Adder("add6", provides='x6', rebind=['x5', 'x4']),
# x7 = x6+x6 = 82
Adder("add7", provides='x7', rebind=['x6', 'x6']))
Adder("add7", provides='x7', rebind=['x6', 'x6']),
)
# Provide the initial variable inputs using a storage dictionary.
store = {
@@ -90,21 +90,19 @@ expected = [
('x7', 82),
]
result = taskflow.engines.run(
flow, engine='serial', store=store)
result = taskflow.engines.run(flow, engine='serial', store=store)
print("Single threaded engine result %s" % result)
for (name, value) in expected:
for name, value in expected:
actual = result.get(name)
if actual != value:
sys.stderr.write(f"{actual} != {value}\n")
unexpected += 1
result = taskflow.engines.run(
flow, engine='parallel', store=store)
result = taskflow.engines.run(flow, engine='parallel', store=store)
print("Multi threaded engine result %s" % result)
for (name, value) in expected:
for name, value in expected:
actual = result.get(name)
if actual != value:
sys.stderr.write(f"{actual} != {value}\n")

57
taskflow/examples/hello_world.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow import engines
@@ -34,6 +34,7 @@ from taskflow import task
# engines using different styles of execution (all can be used to run in
# parallel if a workflow is provided that is parallelizable).
class PrinterTask(task.Task):
def __init__(self, name, show_name=True, inject=None):
super().__init__(name, inject=inject)
@@ -55,26 +56,33 @@ song = lf.Flow("beats")
# singing at once of course!
hi_chorus = uf.Flow('hello')
world_chorus = uf.Flow('world')
for (name, hello, world) in [('bob', 'hello', 'world'),
('joe', 'hellooo', 'worllllld'),
('sue', "helloooooo!", 'wooorllld!')]:
hi_chorus.add(PrinterTask("%s@hello" % name,
# This will show up to the execute() method of
# the task as the argument named 'output' (which
# will allow us to print the character we want).
inject={'output': hello}))
world_chorus.add(PrinterTask("%s@world" % name,
inject={'output': world}))
for name, hello, world in [
('bob', 'hello', 'world'),
('joe', 'hellooo', 'worllllld'),
('sue', "helloooooo!", 'wooorllld!'),
]:
hi_chorus.add(
PrinterTask(
"%s@hello" % name,
# This will show up to the execute() method of
# the task as the argument named 'output' (which
# will allow us to print the character we want).
inject={'output': hello},
)
)
world_chorus.add(PrinterTask("%s@world" % name, inject={'output': world}))
# The composition starts with the conductor and then runs in sequence with
# the chorus running in parallel, but no matter what the 'hello' chorus must
# always run before the 'world' chorus (otherwise the world will fall apart).
song.add(PrinterTask("conductor@begin",
show_name=False, inject={'output': "*ding*"}),
hi_chorus,
world_chorus,
PrinterTask("conductor@end",
show_name=False, inject={'output': "*dong*"}))
song.add(
PrinterTask(
"conductor@begin", show_name=False, inject={'output': "*ding*"}
),
hi_chorus,
world_chorus,
PrinterTask("conductor@end", show_name=False, inject={'output': "*dong*"}),
)
# Run in parallel using eventlet green threads...
try:
@@ -84,22 +92,21 @@ except ImportError:
pass
else:
print("-- Running in parallel using eventlet --")
e = engines.load(song, executor='greenthreaded', engine='parallel',
max_workers=1)
e = engines.load(
song, executor='greenthreaded', engine='parallel', max_workers=1
)
e.run()
# Run in parallel using real threads...
print("-- Running in parallel using threads --")
e = engines.load(song, executor='threaded', engine='parallel',
max_workers=1)
e = engines.load(song, executor='threaded', engine='parallel', max_workers=1)
e.run()
# Run in parallel using external processes...
print("-- Running in parallel using processes --")
e = engines.load(song, executor='processes', engine='parallel',
max_workers=1)
e = engines.load(song, executor='processes', engine='parallel', max_workers=1)
e.run()

17
taskflow/examples/jobboard_produce_consume_colors.py
View File

@@ -23,9 +23,9 @@ import time
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow import exceptions as excp
@@ -121,10 +121,12 @@ def worker(ident, client, consumed):
abandoned_jobs += 1
safe_print(name, "'%s' [abandoned]" % (job))
time.sleep(WORKER_DELAY)
safe_print(name,
"finished (claimed %s jobs, consumed %s jobs,"
" abandoned %s jobs)" % (claimed_jobs, consumed_jobs,
abandoned_jobs), prefix=">>>")
safe_print(
name,
"finished (claimed %s jobs, consumed %s jobs,"
" abandoned %s jobs)" % (claimed_jobs, consumed_jobs, abandoned_jobs),
prefix=">>>",
)
def producer(ident, client):
@@ -149,6 +151,7 @@ def main():
# TODO(harlowja): Hack to make eventlet work right, remove when the
# following is fixed: https://github.com/eventlet/eventlet/issues/230
from taskflow.utils import eventlet_utils as _eu # noqa
try:
import eventlet as _eventlet # noqa
except ImportError:

6
taskflow/examples/parallel_table_multiply.py
View File

@@ -20,9 +20,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import futurist

9
taskflow/examples/persistence_example.py
View File

@@ -21,9 +21,9 @@ import traceback
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)
@@ -95,8 +95,7 @@ with eu.get_backend(backend_uri) as backend:
flow = make_flow(blowup=blowup)
eu.print_wrapped("Running")
try:
eng = engines.load(flow, engine='serial',
backend=backend, book=book)
eng = engines.load(flow, engine='serial', backend=backend, book=book)
eng.run()
if not blowup:
eu.rm_path(persist_path)

28
taskflow/examples/pseudo_scoping.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import taskflow.engines
@@ -43,12 +43,7 @@ from taskflow import task
# his or her phone number from phone book and call.
PHONE_BOOK = {
'jim': '444',
'joe': '555',
'iv_m': '666',
'josh': '777'
}
PHONE_BOOK = {'jim': '444', 'joe': '555', 'iv_m': '666', 'josh': '777'}
class FetchNumberTask(task.Task):
@@ -67,11 +62,10 @@ class CallTask(task.Task):
def execute(self, person, number):
print(f'Calling {person} {number}.')
# This is how it works for one person:
simple_flow = lf.Flow('simple one').add(
FetchNumberTask(),
CallTask())
simple_flow = lf.Flow('simple one').add(FetchNumberTask(), CallTask())
print('Running simple flow:')
taskflow.engines.run(simple_flow, store={'person': 'Josh'})
@@ -85,11 +79,10 @@ def subflow_factory(prefix):
return f'{prefix}-{what}'
return lf.Flow(pr('flow')).add(
FetchNumberTask(pr('fetch'),
provides=pr('number'),
rebind=[pr('person')]),
CallTask(pr('call'),
rebind=[pr('person'), pr('number')])
FetchNumberTask(
pr('fetch'), provides=pr('number'), rebind=[pr('person')]
),
CallTask(pr('call'), rebind=[pr('person'), pr('number')]),
)
@@ -107,5 +100,6 @@ def call_them_all():
flow.add(subflow_factory(prefix))
taskflow.engines.run(flow, store=persons)
print('\nCalling many people using prefixed factory:')
call_them_all()

31
taskflow/examples/resume_from_backend.py
View File

@@ -20,9 +20,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)
@@ -63,8 +63,9 @@ def print_task_states(flowdetail, msg):
print(f"Flow '{flowdetail.name}' state: {flowdetail.state}")
# Sort by these so that our test validation doesn't get confused by the
# order in which the items in the flow detail can be in.
items = sorted((td.name, td.version, td.state, td.results)
for td in flowdetail)
items = sorted(
(td.name, td.version, td.state, td.results) for td in flowdetail
)
for item in items:
print(" %s==%s: %s, result=%s" % item)
@@ -94,17 +95,18 @@ def flow_factory():
return lf.Flow('resume from backend example').add(
TestTask(name='first'),
InterruptTask(name='boom'),
TestTask(name='second'))
TestTask(name='second'),
)
# INITIALIZE PERSISTENCE ####################################
with eu.get_backend() as backend:
# Create a place where the persistence information will be stored.
book = models.LogBook("example")
flow_detail = models.FlowDetail("resume from backend example",
uuid=uuidutils.generate_uuid())
flow_detail = models.FlowDetail(
"resume from backend example", uuid=uuidutils.generate_uuid()
)
book.add(flow_detail)
with contextlib.closing(backend.get_connection()) as conn:
conn.save_logbook(book)
@@ -112,8 +114,9 @@ with eu.get_backend() as backend:
# CREATE AND RUN THE FLOW: FIRST ATTEMPT ####################
flow = flow_factory()
engine = taskflow.engines.load(flow, flow_detail=flow_detail,
book=book, backend=backend)
engine = taskflow.engines.load(
flow, flow_detail=flow_detail, book=book, backend=backend
)
print_task_states(flow_detail, "At the beginning, there is no state")
eu.print_wrapped("Running")
@@ -135,8 +138,8 @@ with eu.get_backend() as backend:
# running the above flow crashes).
flow2 = flow_factory()
flow_detail_2 = find_flow_detail(backend, book.uuid, flow_detail.uuid)
engine2 = taskflow.engines.load(flow2,
flow_detail=flow_detail_2,
backend=backend, book=book)
engine2 = taskflow.engines.load(
flow2, flow_detail=flow_detail_2, backend=backend, book=book
)
engine2.run()
print_task_states(flow_detail_2, "At the end")

12
taskflow/examples/resume_many_flows.py
View File

@@ -42,9 +42,9 @@ def _exec(cmd, add_env=None):
env = os.environ.copy()
env.update(add_env)
proc = subprocess.Popen(cmd, env=env, stdin=None,
stdout=subprocess.PIPE,
stderr=sys.stderr)
proc = subprocess.Popen(
cmd, env=env, stdin=None, stdout=subprocess.PIPE, stderr=sys.stderr
)
stdout, _stderr = proc.communicate()
rc = proc.returncode
@@ -54,8 +54,9 @@ def _exec(cmd, add_env=None):
def _path_to(name):
return os.path.abspath(os.path.join(os.path.dirname(__file__),
'resume_many_flows', name))
return os.path.abspath(
os.path.join(os.path.dirname(__file__), 'resume_many_flows', name)
)
def main():
@@ -87,5 +88,6 @@ def main():
if tmp_path:
example_utils.rm_path(tmp_path)
if __name__ == '__main__':
main()

3
taskflow/examples/resume_many_flows/my_flows.py
View File

@@ -38,4 +38,5 @@ def flow_factory():
return lf.Flow('example').add(
TestTask(name='first'),
UnfortunateTask(name='boom'),
TestTask(name='second'))
TestTask(name='second'),
)

8
taskflow/examples/resume_many_flows/resume_all.py
View File

@@ -20,7 +20,8 @@ logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(
os.path.join(self_dir, os.pardir, os.pardir, os.pardir))
os.path.join(self_dir, os.pardir, os.pardir, os.pardir)
)
example_dir = os.path.abspath(os.path.join(self_dir, os.pardir))
sys.path.insert(0, top_dir)
@@ -38,8 +39,9 @@ FINISHED_STATES = (states.SUCCESS, states.FAILURE, states.REVERTED)
def resume(flowdetail, backend):
print(f'Resuming flow {flowdetail.name} {flowdetail.uuid}')
engine = taskflow.engines.load_from_detail(flow_detail=flowdetail,
backend=backend)
engine = taskflow.engines.load_from_detail(
flow_detail=flowdetail, backend=backend
)
engine.run()

15
taskflow/examples/resume_many_flows/run_flow.py
View File

@@ -20,7 +20,8 @@ logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(
os.path.join(self_dir, os.pardir, os.pardir, os.pardir))
os.path.join(self_dir, os.pardir, os.pardir, os.pardir)
)
example_dir = os.path.abspath(os.path.join(self_dir, os.pardir))
sys.path.insert(0, top_dir)
@@ -34,8 +35,12 @@ import my_flows # noqa
with example_utils.get_backend() as backend:
engine = taskflow.engines.load_from_factory(my_flows.flow_factory,
backend=backend)
print('Running flow {} {}'.format(engine.storage.flow_name,
engine.storage.flow_uuid))
engine = taskflow.engines.load_from_factory(
my_flows.flow_factory, backend=backend
)
print(
'Running flow {} {}'.format(
engine.storage.flow_name, engine.storage.flow_uuid
)
)
engine.run()

55
taskflow/examples/resume_vm_boot.py
View File

@@ -23,9 +23,9 @@ import time
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)
@@ -59,6 +59,7 @@ def slow_down(how_long=0.5):
class PrintText(task.Task):
"""Just inserts some text print outs in a workflow."""
def __init__(self, print_what, no_slow=False):
content_hash = hashlib.md5(print_what.encode('utf-8')).hexdigest()[0:8]
super().__init__(name="Print: %s" % (content_hash))
@@ -75,6 +76,7 @@ class PrintText(task.Task):
class DefineVMSpec(task.Task):
"""Defines a vm specification to be."""
def __init__(self, name):
super().__init__(provides='vm_spec', name=name)
@@ -90,6 +92,7 @@ class DefineVMSpec(task.Task):
class LocateImages(task.Task):
"""Locates where the vm images are."""
def __init__(self, name):
super().__init__(provides='image_locations', name=name)
@@ -103,9 +106,9 @@ class LocateImages(task.Task):
class DownloadImages(task.Task):
"""Downloads all the vm images."""
def __init__(self, name):
super().__init__(provides='download_paths',
name=name)
super().__init__(provides='download_paths', name=name)
def execute(self, image_locations):
for src, loc in image_locations.items():
@@ -116,14 +119,14 @@ class DownloadImages(task.Task):
class CreateNetworkTpl(task.Task):
"""Generates the network settings file to be placed in the images."""
SYSCONFIG_CONTENTS = """DEVICE=eth%s
BOOTPROTO=static
IPADDR=%s
ONBOOT=yes"""
def __init__(self, name):
super().__init__(provides='network_settings',
name=name)
super().__init__(provides='network_settings', name=name)
def execute(self, ips):
settings = []
@@ -134,6 +137,7 @@ ONBOOT=yes"""
class AllocateIP(task.Task):
"""Allocates the ips for the given vm."""
def __init__(self, name):
super().__init__(provides='ips', name=name)
@@ -146,13 +150,15 @@ class AllocateIP(task.Task):
class WriteNetworkSettings(task.Task):
"""Writes all the network settings into the downloaded images."""
def execute(self, download_paths, network_settings):
for j, path in enumerate(download_paths):
with slow_down(1):
print(f"Mounting {path} to /tmp/{j}")
for i, setting in enumerate(network_settings):
filename = ("/tmp/etc/sysconfig/network-scripts/"
"ifcfg-eth%s" % (i))
filename = "/tmp/etc/sysconfig/network-scripts/ifcfg-eth%s" % (
i
)
with slow_down(1):
print("Writing to %s" % (filename))
print(setting)
@@ -160,6 +166,7 @@ class WriteNetworkSettings(task.Task):
class BootVM(task.Task):
"""Fires off the vm boot operation."""
def execute(self, vm_spec):
print("Starting vm!")
with slow_down(1):
@@ -168,6 +175,7 @@ class BootVM(task.Task):
class AllocateVolumes(task.Task):
"""Allocates the volumes for the vm."""
def execute(self, vm_spec):
volumes = []
for i in range(0, vm_spec['volumes']):
@@ -179,6 +187,7 @@ class AllocateVolumes(task.Task):
class FormatVolumes(task.Task):
"""Formats the volumes for the vm."""
def execute(self, volumes):
for v in volumes:
print("Formatting volume %s" % v)
@@ -215,14 +224,15 @@ def create_flow():
),
# Ya it worked!
PrintText("Finished vm create.", no_slow=True),
PrintText("Instance is running!", no_slow=True))
PrintText("Instance is running!", no_slow=True),
)
return flow
eu.print_wrapped("Initializing")
# Setup the persistence & resumption layer.
with eu.get_backend() as backend:
# Try to find a previously passed in tracking id...
try:
book_id, flow_id = sys.argv[2].split("+", 1)
@@ -256,17 +266,24 @@ with eu.get_backend() as backend:
book = models.LogBook("vm-boot")
with contextlib.closing(backend.get_connection()) as conn:
conn.save_logbook(book)
engine = engines.load_from_factory(create_flow,
backend=backend, book=book,
engine='parallel',
executor=executor)
print("!! Your tracking id is: '{}+{}'".format(
book.uuid, engine.storage.flow_uuid))
engine = engines.load_from_factory(
create_flow,
backend=backend,
book=book,
engine='parallel',
executor=executor,
)
print(
"!! Your tracking id is: '{}+{}'".format(
book.uuid, engine.storage.flow_uuid
)
)
print("!! Please submit this on later runs for tracking purposes")
else:
# Attempt to load from a previously partially completed flow.
engine = engines.load_from_detail(flow_detail, backend=backend,
engine='parallel', executor=executor)
engine = engines.load_from_detail(
flow_detail, backend=backend, engine='parallel', executor=executor
)
# Make me my vm please!
eu.print_wrapped('Running')

32
taskflow/examples/resume_volume_create.py
View File

@@ -23,9 +23,9 @@ import time
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)
@@ -90,10 +90,12 @@ class CreateSpecForVolumes(task.Task):
def execute(self):
volumes = []
for i in range(0, random.randint(1, 10)):
volumes.append({
'type': 'disk',
'location': "/dev/vda%s" % (i + 1),
})
volumes.append(
{
'type': 'disk',
'location': "/dev/vda%s" % (i + 1),
}
)
return volumes
@@ -115,7 +117,8 @@ flow = lf.Flow("root").add(
PrintText("I need a nap, it took me a while to build those specs."),
PrepareVolumes(),
),
PrintText("Finished volume create", no_slow=True))
PrintText("Finished volume create", no_slow=True),
)
# Setup the persistence & resumption layer.
with example_utils.get_backend() as backend:
@@ -139,16 +142,19 @@ with example_utils.get_backend() as backend:
book.add(flow_detail)
with contextlib.closing(backend.get_connection()) as conn:
conn.save_logbook(book)
print("!! Your tracking id is: '{}+{}'".format(book.uuid,
flow_detail.uuid))
print(
"!! Your tracking id is: '{}+{}'".format(
book.uuid, flow_detail.uuid
)
)
print("!! Please submit this on later runs for tracking purposes")
else:
flow_detail = find_flow_detail(backend, book_id, flow_id)
# Load and run.
engine = engines.load(flow,
flow_detail=flow_detail,
backend=backend, engine='serial')
engine = engines.load(
flow, flow_detail=flow_detail, backend=backend, engine='serial'
)
engine.run()
# How to use.

16
taskflow/examples/retry_flow.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import taskflow.engines
@@ -52,10 +52,12 @@ class CallJim(task.Task):
# Create your flow and associated tasks (the work to be done).
flow = lf.Flow('retrying-linear',
retry=retry.ParameterizedForEach(
rebind=['phone_directory'],
provides='jim_number')).add(CallJim())
flow = lf.Flow(
'retrying-linear',
retry=retry.ParameterizedForEach(
rebind=['phone_directory'], provides='jim_number'
),
).add(CallJim())
# Now run that flow using the provided initial data (store below).
taskflow.engines.run(flow, store={'phone_directory': [333, 444, 555, 666]})

18
taskflow/examples/reverting_linear.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import taskflow.engines
@@ -61,17 +61,13 @@ class CallSuzzie(task.Task):
# Create your flow and associated tasks (the work to be done).
flow = lf.Flow('simple-linear').add(
CallJim(),
CallJoe(),
CallSuzzie()
)
flow = lf.Flow('simple-linear').add(CallJim(), CallJoe(), CallSuzzie())
try:
# Now run that flow using the provided initial data (store below).
taskflow.engines.run(flow, store=dict(joe_number=444,
jim_number=555,
suzzie_number=666))
taskflow.engines.run(
flow, store=dict(joe_number=444, jim_number=555, suzzie_number=666)
)
except Exception as e:
# NOTE(harlowja): This exception will be the exception that came out of the
# 'CallSuzzie' task instead of a different exception, this is useful since

23
taskflow/examples/run_by_iter.py
View File

@@ -19,9 +19,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)
@@ -51,12 +51,19 @@ def make_alphabet_flow(i):
while ord(curr_value) <= ord(end_value):
next_value = chr(ord(curr_value) + 1)
if curr_value != end_value:
f.add(EchoTask(name="echoer_%s" % curr_value,
rebind={'value': curr_value},
provides=next_value))
f.add(
EchoTask(
name="echoer_%s" % curr_value,
rebind={'value': curr_value},
provides=next_value,
)
)
else:
f.add(EchoTask(name="echoer_%s" % curr_value,
rebind={'value': curr_value}))
f.add(
EchoTask(
name="echoer_%s" % curr_value, rebind={'value': curr_value}
)
)
curr_value = next_value
return f

6
taskflow/examples/run_by_iter_enumerate.py
View File

@@ -19,9 +19,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)

6
taskflow/examples/share_engine_thread.py
View File

@@ -20,9 +20,9 @@ import time
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import futurist

14
taskflow/examples/simple_linear.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import taskflow.engines
@@ -54,11 +54,7 @@ class CallJoe(task.Task):
# Create your flow and associated tasks (the work to be done).
flow = lf.Flow('simple-linear').add(
CallJim(),
CallJoe()
)
flow = lf.Flow('simple-linear').add(CallJim(), CallJoe())
# Now run that flow using the provided initial data (store below).
taskflow.engines.run(flow, store=dict(joe_number=444,
jim_number=555))
taskflow.engines.run(flow, store=dict(joe_number=444, jim_number=555))

21
taskflow/examples/simple_linear_listening.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import taskflow.engines
@@ -82,12 +82,15 @@ flow.add(task.FunctorTask(execute=call_jim))
flow.add(task.FunctorTask(execute=call_joe))
# Now load (but do not run) the flow using the provided initial data.
engine = taskflow.engines.load(flow, store={
'context': {
"joe_number": 444,
"jim_number": 555,
}
})
engine = taskflow.engines.load(
flow,
store={
'context': {
"joe_number": 444,
"jim_number": 555,
}
},
)
# This is where we attach our callback functions to the 2 different
# notification objects that an engine exposes. The usage of a ANY (kleene star)

6
taskflow/examples/simple_linear_pass.py
View File

@@ -19,9 +19,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)

18
taskflow/examples/simple_map_reduce.py
View File

@@ -19,9 +19,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)
@@ -47,7 +47,7 @@ class TotalReducer(task.Task):
def execute(self, *args, **kwargs):
# Reduces all mapped summed outputs into a single value.
total = 0
for (k, v) in kwargs.items():
for k, v in kwargs.items():
# If any other kwargs was passed in, we don't want to use those
# in the calculation of the total...
if k.startswith('reduction_'):
@@ -88,9 +88,13 @@ for i, chunk in enumerate(chunk_iter(CHUNK_SIZE, UPPER_BOUND)):
# The reducer uses all of the outputs of the mappers, so it needs
# to be recorded that it needs access to them (under a specific name).
provided.append("reduction_%s" % i)
mappers.add(SumMapper(name=mapper_name,
rebind={'inputs': mapper_name},
provides=provided[-1]))
mappers.add(
SumMapper(
name=mapper_name,
rebind={'inputs': mapper_name},
provides=provided[-1],
)
)
w.add(mappers)
# The reducer will run last (after all the mappers).

17
taskflow/examples/switch_graph_flow.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow import engines
@@ -56,8 +56,10 @@ print("---------")
print("After run")
print("---------")
backend = e.storage.backend
entries = [os.path.join(backend.memory.root_path, child)
for child in backend.memory.ls(backend.memory.root_path)]
entries = [
os.path.join(backend.memory.root_path, child)
for child in backend.memory.ls(backend.memory.root_path)
]
while entries:
path = entries.pop()
value = backend.memory[path]
@@ -65,5 +67,6 @@ while entries:
print(f"{path} -> {value}")
else:
print("%s" % (path))
entries.extend(os.path.join(path, child)
for child in backend.memory.ls(path))
entries.extend(
os.path.join(path, child) for child in backend.memory.ls(path)
)

6
taskflow/examples/timing_listener.py
View File

@@ -20,9 +20,9 @@ import time
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow import engines

39
taskflow/examples/tox_conductor.py
View File

@@ -25,9 +25,9 @@ import time
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from oslo_utils import timeutils
@@ -128,7 +128,7 @@ def create_review_workflow():
f.add(
MakeTempDir(name="maker"),
RunReview(name="runner"),
CleanResources(name="cleaner")
CleanResources(name="cleaner"),
)
return f
@@ -137,14 +137,16 @@ def generate_reviewer(client, saver, name=NAME):
"""Creates a review producer thread with the given name prefix."""
real_name = "%s_reviewer" % name
no_more = threading.Event()
jb = boards.fetch(real_name, JOBBOARD_CONF,
client=client, persistence=saver)
jb = boards.fetch(
real_name, JOBBOARD_CONF, client=client, persistence=saver
)
def make_save_book(saver, review_id):
# Record what we want to happen (sometime in the future).
book = models.LogBook("book_%s" % review_id)
detail = models.FlowDetail("flow_%s" % review_id,
uuidutils.generate_uuid())
detail = models.FlowDetail(
"flow_%s" % review_id, uuidutils.generate_uuid()
)
book.add(detail)
# Associate the factory method we want to be called (in the future)
# with the book, so that the conductor will be able to call into
@@ -157,8 +159,9 @@ def generate_reviewer(client, saver, name=NAME):
# workflow that represents this review).
factory_args = ()
factory_kwargs = {}
engines.save_factory_details(detail, create_review_workflow,
factory_args, factory_kwargs)
engines.save_factory_details(
detail, create_review_workflow, factory_args, factory_kwargs
)
with contextlib.closing(saver.get_connection()) as conn:
conn.save_logbook(book)
return book
@@ -177,9 +180,11 @@ def generate_reviewer(client, saver, name=NAME):
}
job_name = "{}_{}".format(real_name, review['id'])
print("Posting review '%s'" % review['id'])
jb.post(job_name,
book=make_save_book(saver, review['id']),
details=details)
jb.post(
job_name,
book=make_save_book(saver, review['id']),
details=details,
)
time.sleep(REVIEW_CREATION_DELAY)
# Return the unstarted thread, and a callback that can be used
@@ -190,10 +195,10 @@ def generate_reviewer(client, saver, name=NAME):
def generate_conductor(client, saver, name=NAME):
"""Creates a conductor thread with the given name prefix."""
real_name = "%s_conductor" % name
jb = boards.fetch(name, JOBBOARD_CONF,
client=client, persistence=saver)
conductor = conductors.fetch("blocking", real_name, jb,
engine='parallel', wait_timeout=SCAN_DELAY)
jb = boards.fetch(name, JOBBOARD_CONF, client=client, persistence=saver)
conductor = conductors.fetch(
"blocking", real_name, jb, engine='parallel', wait_timeout=SCAN_DELAY
)
def run():
jb.connect()

6
taskflow/examples/wbe_event_sender.py
View File

@@ -18,9 +18,9 @@ import string
import sys
import time
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow import engines

47
taskflow/examples/wbe_mandelbrot.py
View File

@@ -17,9 +17,9 @@ import math
import os
import sys
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow import engines
@@ -118,7 +118,7 @@ def calculate(engine_conf):
'mandelbrot_config': [-2.0, 1.0, -1.0, 1.0, MAX_ITERATIONS],
'image_config': {
'size': IMAGE_SIZE,
}
},
}
# We need the task names to be in the right order so that we can extract
@@ -135,13 +135,16 @@ def calculate(engine_conf):
# Break the calculation up into chunk size pieces.
rows = [i * chunk_size, i * chunk_size + chunk_size]
flow.add(
MandelCalculator(task_name,
# This ensures the storage symbol with name
# 'chunk_name' is sent into the tasks local
# symbol 'chunk'. This is how we give each
# calculator its own correct sequence of rows
# to work on.
rebind={'chunk': chunk_name}))
MandelCalculator(
task_name,
# This ensures the storage symbol with name
# 'chunk_name' is sent into the tasks local
# symbol 'chunk'. This is how we give each
# calculator its own correct sequence of rows
# to work on.
rebind={'chunk': chunk_name},
)
)
store[chunk_name] = rows
task_names.append(task_name)
@@ -161,9 +164,11 @@ def calculate(engine_conf):
def write_image(results, output_filename=None):
print("Gathered %s results that represents a mandelbrot"
" image (using %s chunks that are computed jointly"
" by %s workers)." % (len(results), CHUNK_COUNT, WORKERS))
print(
"Gathered %s results that represents a mandelbrot"
" image (using %s chunks that are computed jointly"
" by %s workers)." % (len(results), CHUNK_COUNT, WORKERS)
)
if not output_filename:
return
@@ -198,12 +203,14 @@ def create_fractal():
# Setup our transport configuration and merge it into the worker and
# engine configuration so that both of those use it correctly.
shared_conf = dict(BASE_SHARED_CONF)
shared_conf.update({
'transport': 'memory',
'transport_options': {
'polling_interval': 0.1,
},
})
shared_conf.update(
{
'transport': 'memory',
'transport_options': {
'polling_interval': 0.1,
},
}
)
if len(sys.argv) >= 2:
output_filename = sys.argv[1]

51
taskflow/examples/wbe_simple_linear.py
View File

@@ -18,9 +18,9 @@ import os
import sys
import tempfile
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow import engines
@@ -64,7 +64,7 @@ WORKER_CONF = {
# not want to allow all python code to be executed).
'tasks': [
'taskflow.tests.utils:TaskOneArgOneReturn',
'taskflow.tests.utils:TaskMultiArgOneReturn'
'taskflow.tests.utils:TaskMultiArgOneReturn',
],
}
@@ -72,11 +72,14 @@ WORKER_CONF = {
def run(engine_options):
flow = lf.Flow('simple-linear').add(
utils.TaskOneArgOneReturn(provides='result1'),
utils.TaskMultiArgOneReturn(provides='result2')
utils.TaskMultiArgOneReturn(provides='result2'),
)
eng = engines.load(
flow,
store=dict(x=111, y=222, z=333),
engine='worker-based',
**engine_options,
)
eng = engines.load(flow,
store=dict(x=111, y=222, z=333),
engine='worker-based', **engine_options)
eng.run()
return eng.storage.fetch_all()
@@ -92,22 +95,26 @@ if __name__ == "__main__":
if USE_FILESYSTEM:
worker_count = FILE_WORKERS
tmp_path = tempfile.mkdtemp(prefix='wbe-example-')
shared_conf.update({
'transport': 'filesystem',
'transport_options': {
'data_folder_in': tmp_path,
'data_folder_out': tmp_path,
'polling_interval': 0.1,
},
})
shared_conf.update(
{
'transport': 'filesystem',
'transport_options': {
'data_folder_in': tmp_path,
'data_folder_out': tmp_path,
'polling_interval': 0.1,
},
}
)
else:
worker_count = MEMORY_WORKERS
shared_conf.update({
'transport': 'memory',
'transport_options': {
'polling_interval': 0.1,
},
})
shared_conf.update(
{
'transport': 'memory',
'transport_options': {
'polling_interval': 0.1,
},
}
)
worker_conf = dict(WORKER_CONF)
worker_conf.update(shared_conf)
engine_options = dict(shared_conf)

23
taskflow/examples/wrapped_exception.py
View File

@@ -20,9 +20,9 @@ import time
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
@@ -84,14 +84,10 @@ def run(**store):
# here and based on those kwargs it will behave in a different manner
# while executing; this allows for the calling code (see below) to show
# different usages of the failure catching and handling mechanism.
flow = uf.Flow('flow').add(
FirstTask(),
SecondTask()
)
flow = uf.Flow('flow').add(FirstTask(), SecondTask())
try:
with utils.wrap_all_failures():
taskflow.engines.run(flow, store=store,
engine='parallel')
taskflow.engines.run(flow, store=store, engine='parallel')
except exceptions.WrappedFailure as ex:
unknown_failures = []
for a_failure in ex:
@@ -106,20 +102,17 @@ def run(**store):
eu.print_wrapped("Raise and catch first exception only")
run(sleep1=0.0, raise1=True,
sleep2=0.0, raise2=False)
run(sleep1=0.0, raise1=True, sleep2=0.0, raise2=False)
# NOTE(imelnikov): in general, sleeping does not guarantee that we'll have both
# task running before one of them fails, but with current implementation this
# works most of times, which is enough for our purposes here (as an example).
eu.print_wrapped("Raise and catch both exceptions")
run(sleep1=1.0, raise1=True,
sleep2=1.0, raise2=True)
run(sleep1=1.0, raise1=True, sleep2=1.0, raise2=True)
eu.print_wrapped("Handle one exception, and re-raise another")
try:
run(sleep1=1.0, raise1=True,
sleep2=1.0, raise2='boom')
run(sleep1=1.0, raise1=True, sleep2=1.0, raise2='boom')
except TypeError as ex:
print("As expected, TypeError is here: %s" % ex)
else:

23
taskflow/exceptions.py
View File

@@ -64,6 +64,7 @@ class TaskFlowException(Exception):
creating a chain of exceptions for versions of python where
this is not yet implemented/supported natively.
"""
def __init__(self, message, cause=None):
super().__init__(message)
self._cause = cause
@@ -84,8 +85,10 @@ class TaskFlowException(Exception):
def pformat(self, indent=2, indent_text=" ", show_root_class=False):
"""Pretty formats a taskflow exception + any connected causes."""
if indent < 0:
raise ValueError("Provided 'indent' must be greater than"
" or equal to zero instead of %s" % indent)
raise ValueError(
"Provided 'indent' must be greater than"
" or equal to zero instead of %s" % indent
)
buf = io.StringIO()
if show_root_class:
buf.write(reflection.get_class_name(self, fully_qualified=False))
@@ -99,8 +102,9 @@ class TaskFlowException(Exception):
buf.write(os.linesep)
if isinstance(next_up, TaskFlowException):
buf.write(indent_text * active_indent)
buf.write(reflection.get_class_name(next_up,