openstack/taskflow
Code Issues Proposed changes

Add ruff

Browse Source 
Change-Id: I2518e0cf928210acf9cfb2e5f4c19f973df64485
Signed-off-by: Stephen Finucane <stephenfin@redhat.com>
This commit is contained in:
Stephen Finucane
2026-03-11 10:43:22 +00:00
parent e2cabccf61
commit 2a6420a3ed
195 changed files with 9216 additions and 6116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
.pre-commit-config.yaml
View File

@@ -12,14 +12,15 @@ repos:
- id: debug-statements
- id: check-yaml
files: .*\.(yaml|yml)$
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.15.5
hooks:
- id: ruff-check
args: ['--fix', '--unsafe-fixes']
- id: ruff-format
- repo: https://opendev.org/openstack/hacking
rev: 8.0.0
hooks:
- id: hacking
additional_dependencies: []
exclude: '^(doc|releasenotes|tools)/.*$'
- repo: https://github.com/asottile/pyupgrade
rev: v3.21.2
hooks:
- id: pyupgrade
args: [--py310-plus]

2
doc/source/conf.py
View File

@@ -25,7 +25,7 @@ extensions = [
'sphinx.ext.extlinks',
'sphinx.ext.inheritance_diagram',
'sphinx.ext.viewcode',
'openstackdocstheme'
'openstackdocstheme',
]
# openstackdocstheme options

22
pyproject.toml
View File

@@ -95,7 +95,21 @@ parallel = "taskflow.engines.action_engine.engine:ParallelActionEngine"
worker-based = "taskflow.engines.worker_based.engine:WorkerBasedActionEngine"
workers = "taskflow.engines.worker_based.engine:WorkerBasedActionEngine"
[tool.setuptools]
packages = [
"taskflow"
]
[tool.setuptools.packages.find]
include = ["taskflow"]
[tool.ruff]
line-length = 79
[tool.ruff.format]
quote-style = "preserve"
docstring-code-format = true
[tool.ruff.lint]
select = ["E4", "E5", "E7", "E9", "F", "G", "LOG", "S"]
external = ["H"]
ignore = ["E402", "E721", "E731", "E741"]
[tool.ruff.lint.per-file-ignores]
"taskflow/examples/*" = ["S"]
"taskflow/tests/*" = ["S"]

10
releasenotes/source/conf.py
View File

@@ -88,9 +88,13 @@ html_static_path = ['_static']
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
('index', 'taskflowReleaseNotes.tex',
'taskflow Release Notes Documentation',
'taskflow Developers', 'manual'),
(
'index',
'taskflowReleaseNotes.tex',
'taskflow Release Notes Documentation',
'taskflow Developers',
'manual',
),
]

4
setup.py
View File

@@ -15,6 +15,4 @@
import setuptools
setuptools.setup(
setup_requires=['pbr>=2.0.0'],
pbr=True)
setuptools.setup(setup_requires=['pbr>=2.0.0'], pbr=True)

92
taskflow/atom.py
View File

@@ -52,8 +52,9 @@ def _save_as_to_mapping(save_as):
# NOTE(harlowja): this means that your atom will return a indexable
# object, like a list or tuple and the results can be mapped by index
# to that tuple/list that is returned for others to use.
return collections.OrderedDict((key, num)
for num, key in enumerate(save_as))
return collections.OrderedDict(
(key, num) for num, key in enumerate(save_as)
)
elif isinstance(save_as, _set_types):
# NOTE(harlowja): in the case where a set is given we will not be
# able to determine the numeric ordering in a reliable way (since it
@@ -61,8 +62,10 @@ def _save_as_to_mapping(save_as):
# result of the atom will be via the key itself.
return collections.OrderedDict((key, key) for key in save_as)
else:
raise TypeError('Atom provides parameter '
'should be str, set or tuple/list, not %r' % save_as)
raise TypeError(
'Atom provides parameter '
'should be str, set or tuple/list, not %r' % save_as
)
def _build_rebind_dict(req_args, rebind_args):
@@ -84,17 +87,19 @@ def _build_rebind_dict(req_args, rebind_args):
# Extra things were rebound, that may be because of *args
# or **kwargs (or some other reason); so just keep all of them
# using 1:1 rebinding...
rebind.update((a, a) for a in rebind_args[len(req_args):])
rebind.update((a, a) for a in rebind_args[len(req_args) :])
return rebind
elif isinstance(rebind_args, dict):
return rebind_args
else:
raise TypeError("Invalid rebind value '%s' (%s)"
% (rebind_args, type(rebind_args)))
raise TypeError(
"Invalid rebind value '%s' (%s)" % (rebind_args, type(rebind_args))
)
def _build_arg_mapping(atom_name, reqs, rebind_args, function, do_infer,
ignore_list=None):
def _build_arg_mapping(
atom_name, reqs, rebind_args, function, do_infer, ignore_list=None
):
"""Builds an input argument mapping for a given function.
Given a function, its requirements and a rebind mapping this helper
@@ -135,8 +140,9 @@ def _build_arg_mapping(atom_name, reqs, rebind_args, function, do_infer,
# Determine if there are optional arguments that we may or may not take.
if do_infer:
opt_args = sets.OrderedSet(all_args)
opt_args = opt_args - set(itertools.chain(required.keys(),
iter(ignore_list)))
opt_args = opt_args - set(
itertools.chain(required.keys(), iter(ignore_list))
)
optional = collections.OrderedDict((a, a) for a in opt_args)
else:
optional = collections.OrderedDict()
@@ -146,14 +152,17 @@ def _build_arg_mapping(atom_name, reqs, rebind_args, function, do_infer,
extra_args = sets.OrderedSet(required.keys())
extra_args -= all_args
if extra_args:
raise ValueError('Extra arguments given to atom %s: %s'
% (atom_name, list(extra_args)))
raise ValueError(
'Extra arguments given to atom %s: %s'
% (atom_name, list(extra_args))
)
# NOTE(imelnikov): don't use set to preserve order in error message
missing_args = [arg for arg in req_args if arg not in required]
if missing_args:
raise ValueError('Missing arguments for atom %s: %s'
% (atom_name, missing_args))
raise ValueError(
'Missing arguments for atom %s: %s' % (atom_name, missing_args)
)
return required, optional
@@ -244,9 +253,18 @@ class Atom(metaclass=abc.ABCMeta):
default_provides = None
def __init__(self, name=None, provides=None, requires=None,
auto_extract=True, rebind=None, inject=None,
ignore_list=None, revert_rebind=None, revert_requires=None):
def __init__(
self,
name=None,
provides=None,
requires=None,
auto_extract=True,
rebind=None,
inject=None,
ignore_list=None,
revert_rebind=None,
revert_requires=None,
):
if provides is None:
provides = self.default_provides
@@ -263,8 +281,9 @@ class Atom(metaclass=abc.ABCMeta):
self.rebind, exec_requires, self.optional = self._build_arg_mapping(
self.execute,
requires=requires,
rebind=rebind, auto_extract=auto_extract,
ignore_list=ignore_list
rebind=rebind,
auto_extract=auto_extract,
ignore_list=ignore_list,
)
revert_ignore = ignore_list + list(_default_revert_args)
@@ -273,10 +292,11 @@ class Atom(metaclass=abc.ABCMeta):
requires=revert_requires or requires,
rebind=revert_rebind or rebind,
auto_extract=auto_extract,
ignore_list=revert_ignore
ignore_list=revert_ignore,
)
(self.revert_rebind, addl_requires, self.revert_optional) = (
revert_mapping
)
(self.revert_rebind, addl_requires,
self.revert_optional) = revert_mapping
# TODO(bnemec): This should be documented as an ivar, but can't be due
# to https://github.com/sphinx-doc/sphinx/issues/2549
@@ -284,18 +304,30 @@ class Atom(metaclass=abc.ABCMeta):
#: requires to function.
self.requires = exec_requires.union(addl_requires)
def _build_arg_mapping(self, executor, requires=None, rebind=None,
auto_extract=True, ignore_list=None):
def _build_arg_mapping(
self,
executor,
requires=None,
rebind=None,
auto_extract=True,
ignore_list=None,
):
required, optional = _build_arg_mapping(self.name, requires, rebind,
executor, auto_extract,
ignore_list=ignore_list)
required, optional = _build_arg_mapping(
self.name,
requires,
rebind,
executor,
auto_extract,
ignore_list=ignore_list,
)
# Form the real rebind mapping, if a key name is the same as the
# key value, then well there is no rebinding happening, otherwise
# there will be.
rebind = collections.OrderedDict()
for (arg_name, bound_name) in itertools.chain(required.items(),
optional.items()):
for arg_name, bound_name in itertools.chain(
required.items(), optional.items()
):
rebind.setdefault(arg_name, bound_name)
requires = sets.OrderedSet(required.values())
optional = sets.OrderedSet(optional.values())

6
taskflow/conductors/backends/__init__.py
View File

@@ -34,10 +34,12 @@ def fetch(kind, name, jobboard, namespace=CONDUCTOR_NAMESPACE, **kwargs):
LOG.debug('Looking for %r conductor driver in %r', kind, namespace)
try:
mgr = stevedore.driver.DriverManager(
namespace, kind,
namespace,
kind,
invoke_on_load=True,
invoke_args=(name, jobboard),
invoke_kwds=kwargs)
invoke_kwds=kwargs,
)
return mgr.driver
except RuntimeError as e:
raise exc.NotFound("Could not find conductor %s" % (kind), e)

27
taskflow/conductors/backends/impl_blocking.py
View File

@@ -27,13 +27,24 @@ class BlockingConductor(impl_executor.ExecutorConductor):
def _executor_factory():
return futurist.SynchronousExecutor()
def __init__(self, name, jobboard,
persistence=None, engine=None,
engine_options=None, wait_timeout=None,
log=None, max_simultaneous_jobs=MAX_SIMULTANEOUS_JOBS):
def __init__(
self,
name,
jobboard,
persistence=None,
engine=None,
engine_options=None,
wait_timeout=None,
log=None,
max_simultaneous_jobs=MAX_SIMULTANEOUS_JOBS,
):
super().__init__(
name, jobboard,
persistence=persistence, engine=engine,
name,
jobboard,
persistence=persistence,
engine=engine,
engine_options=engine_options,
wait_timeout=wait_timeout, log=log,
max_simultaneous_jobs=max_simultaneous_jobs)
wait_timeout=wait_timeout,
log=log,
max_simultaneous_jobs=max_simultaneous_jobs,
)

197
taskflow/conductors/backends/impl_executor.py
View File

@@ -78,47 +78,72 @@ class ExecutorConductor(base.Conductor, metaclass=abc.ABCMeta):
"""
#: Exceptions that will **not** cause consumption to occur.
NO_CONSUME_EXCEPTIONS = tuple([
excp.ExecutionFailure,
excp.StorageFailure,
])
NO_CONSUME_EXCEPTIONS = tuple(
[
excp.ExecutionFailure,
excp.StorageFailure,
]
)
_event_factory = threading.Event
"""This attribute *can* be overridden by subclasses (for example if
an eventlet *green* event works better for the conductor user)."""
EVENTS_EMITTED = tuple([
'compilation_start', 'compilation_end',
'preparation_start', 'preparation_end',
'validation_start', 'validation_end',
'running_start', 'running_end',
'job_consumed', 'job_abandoned',
])
EVENTS_EMITTED = tuple(
[
'compilation_start',
'compilation_end',
'preparation_start',
'preparation_end',
'validation_start',
'validation_end',
'running_start',
'running_end',
'job_consumed',
'job_abandoned',
]
)
"""Events will be emitted for each of the events above. The event is
emitted to listeners registered with the conductor.
"""
def __init__(self, name, jobboard,
persistence=None, engine=None,
engine_options=None, wait_timeout=None,
log=None, max_simultaneous_jobs=MAX_SIMULTANEOUS_JOBS):
def __init__(
self,
name,
jobboard,
persistence=None,
engine=None,
engine_options=None,
wait_timeout=None,
log=None,
max_simultaneous_jobs=MAX_SIMULTANEOUS_JOBS,
):
super().__init__(
name, jobboard, persistence=persistence,
engine=engine, engine_options=engine_options)
name,
jobboard,
persistence=persistence,
engine=engine,
engine_options=engine_options,
)
self._wait_timeout = tt.convert_to_timeout(
value=wait_timeout, default_value=self.WAIT_TIMEOUT,
event_factory=self._event_factory)
value=wait_timeout,
default_value=self.WAIT_TIMEOUT,
event_factory=self._event_factory,
)
self._dead = self._event_factory()
self._log = misc.pick_first_not_none(log, self.LOG, LOG)
self._max_simultaneous_jobs = int(
misc.pick_first_not_none(max_simultaneous_jobs,
self.MAX_SIMULTANEOUS_JOBS))
misc.pick_first_not_none(
max_simultaneous_jobs, self.MAX_SIMULTANEOUS_JOBS
)
)
self._dispatched = set()
def _executor_factory(self):
"""Creates an executor to be used during dispatching."""
raise excp.NotImplementedError("This method must be implemented but"
" it has not been")
raise excp.NotImplementedError(
"This method must be implemented but it has not been"
)
def stop(self):
self._wait_timeout.interrupt()
@@ -134,8 +159,9 @@ class ExecutorConductor(base.Conductor, metaclass=abc.ABCMeta):
def _listeners_from_job(self, job, engine):
listeners = super()._listeners_from_job(job, engine)
listeners.append(logging_listener.LoggingListener(engine,
log=self._log))
listeners.append(
logging_listener.LoggingListener(engine, log=self._log)
)
return listeners
def _dispatch_job(self, job):
@@ -156,17 +182,22 @@ class ExecutorConductor(base.Conductor, metaclass=abc.ABCMeta):
has_suspended = False
for _state in engine.run_iter():
if not has_suspended and self._wait_timeout.is_stopped():
self._log.info("Conductor stopped, requesting "
"suspension of engine running "
"job %s", job)
self._log.info(
"Conductor stopped, requesting "
"suspension of engine running "
"job %s",
job,
)
engine.suspend()
has_suspended = True
try:
for stage_func, event_name in [(engine.compile, 'compilation'),
(engine.prepare, 'preparation'),
(engine.validate, 'validation'),
(_run_engine, 'running')]:
for stage_func, event_name in [
(engine.compile, 'compilation'),
(engine.prepare, 'preparation'),
(engine.validate, 'validation'),
(_run_engine, 'running'),
]:
self._notifier.notify("%s_start" % event_name, details)
stage_func()
self._notifier.notify("%s_end" % event_name, details)
@@ -177,23 +208,35 @@ class ExecutorConductor(base.Conductor, metaclass=abc.ABCMeta):
if consume:
self._log.warning(
"Job execution failed (consumption being"
" skipped): %s [%s failures]", job, len(e))
" skipped): %s [%s failures]",
job,
len(e),
)
else:
self._log.warning(
"Job execution failed (consumption"
" proceeding): %s [%s failures]", job, len(e))
" proceeding): %s [%s failures]",
job,
len(e),
)
# Show the failure/s + traceback (if possible)...
for i, f in enumerate(e):
self._log.warning("%s. %s", i + 1,
f.pformat(traceback=True))
self._log.warning(
"%s. %s", i + 1, f.pformat(traceback=True)
)
except self.NO_CONSUME_EXCEPTIONS:
self._log.warning("Job execution failed (consumption being"
" skipped): %s", job, exc_info=True)
self._log.warning(
"Job execution failed (consumption being skipped): %s",
job,
exc_info=True,
)
consume = False
except Exception:
self._log.warning(
"Job execution failed (consumption proceeding): %s",
job, exc_info=True)
job,
exc_info=True,
)
else:
if engine.storage.get_flow_state() == states.SUSPENDED:
self._log.info("Job execution was suspended: %s", job)
@@ -206,32 +249,43 @@ class ExecutorConductor(base.Conductor, metaclass=abc.ABCMeta):
try:
if consume:
self._jobboard.consume(job, self._name)
self._notifier.notify("job_consumed", {
'job': job,
'conductor': self,
'persistence': self._persistence,
})
self._notifier.notify(
"job_consumed",
{
'job': job,
'conductor': self,
'persistence': self._persistence,
},
)
elif trash:
self._jobboard.trash(job, self._name)
self._notifier.notify("job_trashed", {
'job': job,
'conductor': self,
'persistence': self._persistence,
})
self._notifier.notify(
"job_trashed",
{
'job': job,
'conductor': self,
'persistence': self._persistence,
},
)
else:
self._jobboard.abandon(job, self._name)
self._notifier.notify("job_abandoned", {
'job': job,
'conductor': self,
'persistence': self._persistence,
})
self._notifier.notify(
"job_abandoned",
{
'job': job,
'conductor': self,
'persistence': self._persistence,
},
)
except (excp.JobFailure, excp.NotFound):
if consume:
self._log.warn("Failed job consumption: %s", job,
exc_info=True)
self._log.warn(
"Failed job consumption: %s", job, exc_info=True
)
else:
self._log.warn("Failed job abandonment: %s", job,
exc_info=True)
self._log.warn(
"Failed job abandonment: %s", job, exc_info=True
)
def _on_job_done(self, job, fut):
consume = False
@@ -273,7 +327,8 @@ class ExecutorConductor(base.Conductor, metaclass=abc.ABCMeta):
if max_dispatches == 0:
raise StopIteration
fresh_period = timeutils.StopWatch(
duration=self.REFRESH_PERIODICITY)
duration=self.REFRESH_PERIODICITY
)
fresh_period.start()
while not is_stopped():
any_dispatched = False
@@ -284,28 +339,32 @@ class ExecutorConductor(base.Conductor, metaclass=abc.ABCMeta):
ensure_fresh = False
job_it = itertools.takewhile(
self._can_claim_more_jobs,
self._jobboard.iterjobs(ensure_fresh=ensure_fresh))
self._jobboard.iterjobs(ensure_fresh=ensure_fresh),
)
for job in job_it:
self._log.debug("Trying to claim job: %s", job)
try:
self._jobboard.claim(job, self._name)
except (excp.UnclaimableJob, excp.NotFound):
self._log.debug("Job already claimed or"
" consumed: %s", job)
self._log.debug(
"Job already claimed or consumed: %s", job
)
else:
try:
fut = executor.submit(self._dispatch_job, job)
except RuntimeError:
with excutils.save_and_reraise_exception():
self._log.warn("Job dispatch submitting"
" failed: %s", job)
self._log.warn(
"Job dispatch submitting failed: %s", job
)
self._try_finish_job(job, False)
else:
fut.job = job
self._dispatched.add(fut)
any_dispatched = True
fut.add_done_callback(
functools.partial(self._on_job_done, job))
functools.partial(self._on_job_done, job)
)
total_dispatched = next(dispatch_gen)
if not any_dispatched and not is_stopped():
self._wait_timeout.wait()
@@ -314,8 +373,9 @@ class ExecutorConductor(base.Conductor, metaclass=abc.ABCMeta):
# max dispatch number (which implies we should do no more work).
with excutils.save_and_reraise_exception():
if max_dispatches >= 0 and total_dispatched >= max_dispatches:
self._log.info("Maximum dispatch limit of %s reached",
max_dispatches)
self._log.info(
"Maximum dispatch limit of %s reached", max_dispatches
)
def run(self, max_dispatches=None):
self._dead.clear()
@@ -323,8 +383,7 @@ class ExecutorConductor(base.Conductor, metaclass=abc.ABCMeta):
try:
self._jobboard.register_entity(self.conductor)
with self._executor_factory() as executor:
self._run_until_dead(executor,
max_dispatches=max_dispatches)
self._run_until_dead(executor, max_dispatches=max_dispatches)
except StopIteration:
pass
except KeyboardInterrupt:

36
taskflow/conductors/backends/impl_nonblocking.py
View File

@@ -47,20 +47,34 @@ class NonBlockingConductor(impl_executor.ExecutorConductor):
max_workers = max_simultaneous_jobs
return futurist.ThreadPoolExecutor(max_workers=max_workers)
def __init__(self, name, jobboard,
persistence=None, engine=None,
engine_options=None, wait_timeout=None,
log=None, max_simultaneous_jobs=MAX_SIMULTANEOUS_JOBS,
executor_factory=None):
def __init__(
self,
name,
jobboard,
persistence=None,
engine=None,
engine_options=None,
wait_timeout=None,
log=None,
max_simultaneous_jobs=MAX_SIMULTANEOUS_JOBS,
executor_factory=None,
):
super().__init__(
name, jobboard,
persistence=persistence, engine=engine,
engine_options=engine_options, wait_timeout=wait_timeout,
log=log, max_simultaneous_jobs=max_simultaneous_jobs)
name,
jobboard,
persistence=persistence,
engine=engine,
engine_options=engine_options,
wait_timeout=wait_timeout,
log=log,
max_simultaneous_jobs=max_simultaneous_jobs,
)
if executor_factory is None:
self._executor_factory = self._default_executor_factory
else:
if not callable(executor_factory):
raise ValueError("Provided keyword argument 'executor_factory'"
" must be callable")
raise ValueError(
"Provided keyword argument 'executor_factory'"
" must be callable"
)
self._executor_factory = executor_factory

35
taskflow/conductors/base.py
View File

@@ -37,8 +37,14 @@ class Conductor(metaclass=abc.ABCMeta):
#: Entity kind used when creating new entity objects
ENTITY_KIND = 'conductor'
def __init__(self, name, jobboard,
persistence=None, engine=None, engine_options=None):
def __init__(
self,
name,
jobboard,
persistence=None,
engine=None,
engine_options=None,
):
self._name = name
self._jobboard = jobboard
self._engine = engine
@@ -91,9 +97,11 @@ class Conductor(metaclass=abc.ABCMeta):
flow_uuid = job.details["flow_uuid"]
flow_detail = book.find(flow_uuid)
if flow_detail is None:
raise excp.NotFound("No matching flow detail found in"
" jobs book for flow detail"
" with uuid %s" % flow_uuid)
raise excp.NotFound(
"No matching flow detail found in"
" jobs book for flow detail"
" with uuid %s" % flow_uuid
)
else:
choices = len(book)
if choices == 1:
@@ -101,8 +109,10 @@ class Conductor(metaclass=abc.ABCMeta):
elif choices == 0:
raise excp.NotFound("No flow detail(s) found in jobs book")
else:
raise excp.MultipleChoices("No matching flow detail found (%s"
" choices) in jobs book" % choices)
raise excp.MultipleChoices(
"No matching flow detail found (%s"
" choices) in jobs book" % choices
)
return flow_detail
def _engine_from_job(self, job):
@@ -116,10 +126,13 @@ class Conductor(metaclass=abc.ABCMeta):
if job.details and 'store' in job.details:
store.update(job.details["store"])
engine = engines.load_from_detail(flow_detail, store=store,
engine=self._engine,
backend=self._persistence,
**self._engine_options)
engine = engines.load_from_detail(
flow_detail,
store=store,
engine=self._engine,
backend=self._persistence,
**self._engine_options,
)
return engine
def _listeners_from_job(self, job, engine):

25
taskflow/deciders.py
View File

@@ -71,23 +71,32 @@ class Depth(misc.StrEnum):
# Nothing to do in the first place...
return desired_depth
if not isinstance(desired_depth, str):
raise TypeError("Unexpected desired depth type, string type"
" expected, not %s" % type(desired_depth))
raise TypeError(
"Unexpected desired depth type, string type"
" expected, not %s" % type(desired_depth)
)
try:
return cls(desired_depth.upper())
except ValueError:
pretty_depths = sorted([a_depth.name for a_depth in cls])
raise ValueError("Unexpected decider depth value, one of"
" %s (case-insensitive) is expected and"
" not '%s'" % (pretty_depths, desired_depth))
raise ValueError(
"Unexpected decider depth value, one of"
" %s (case-insensitive) is expected and"
" not '%s'" % (pretty_depths, desired_depth)
)
# Depth area of influence order (from greater influence to least).
#
# Order very much matters here...
_ORDERING = tuple([
Depth.ALL, Depth.FLOW, Depth.NEIGHBORS, Depth.ATOM,
])
_ORDERING = tuple(
[
Depth.ALL,
Depth.FLOW,
Depth.NEIGHBORS,
Depth.ATOM,
]
)
def pick_widest(depths):

11
taskflow/engines/__init__.py
View File

@@ -18,13 +18,14 @@ from oslo_utils import eventletutils as _eventletutils
# are highly recommended to be patched (or otherwise bad things could
# happen).
_eventletutils.warn_eventlet_not_patched(
expected_patched_modules=['time', 'thread'])
expected_patched_modules=['time', 'thread']
)
# Promote helpers to this module namespace (for easy access).
from taskflow.engines.helpers import flow_from_detail # noqa
from taskflow.engines.helpers import load # noqa
from taskflow.engines.helpers import load_from_detail # noqa
from taskflow.engines.helpers import flow_from_detail # noqa
from taskflow.engines.helpers import load # noqa
from taskflow.engines.helpers import load_from_detail # noqa
from taskflow.engines.helpers import load_from_factory # noqa
from taskflow.engines.helpers import run # noqa
from taskflow.engines.helpers import run # noqa
from taskflow.engines.helpers import save_factory_details # noqa

8
taskflow/engines/action_engine/actions/base.py
View File

@@ -26,8 +26,12 @@ class Action(metaclass=abc.ABCMeta):
"""
#: States that are expected to have a result to save...
SAVE_RESULT_STATES = (states.SUCCESS, states.FAILURE,
states.REVERTED, states.REVERT_FAILURE)
SAVE_RESULT_STATES = (
states.SUCCESS,
states.FAILURE,
states.REVERTED,
states.REVERT_FAILURE,
)
def __init__(self, storage, notifier):
self._storage = storage

10
taskflow/engines/action_engine/actions/retry.py
View File

@@ -30,13 +30,13 @@ class RetryAction(base.Action):
arguments = self._storage.fetch_mapped_args(
retry.revert_rebind,
atom_name=retry.name,
optional_args=retry.revert_optional
optional_args=retry.revert_optional,
)
else:
arguments = self._storage.fetch_mapped_args(
retry.rebind,
atom_name=retry.name,
optional_args=retry.optional
optional_args=retry.optional,
)
history = self._storage.get_retry_history(retry.name)
arguments[retry_atom.EXECUTE_REVERT_HISTORY] = history
@@ -74,7 +74,8 @@ class RetryAction(base.Action):
def schedule_execution(self, retry):
self.change_state(retry, states.RUNNING)
return self._retry_executor.execute_retry(
retry, self._get_retry_args(retry))
retry, self._get_retry_args(retry)
)
def complete_reversion(self, retry, result):
if isinstance(result, failure.Failure):
@@ -94,7 +95,8 @@ class RetryAction(base.Action):
retry_atom.REVERT_FLOW_FAILURES: self._storage.get_failures(),
}
return self._retry_executor.revert_retry(
retry, self._get_retry_args(retry, addons=arg_addons, revert=True))
retry, self._get_retry_args(retry, addons=arg_addons, revert=True)
)
def on_failure(self, retry, atom, last_failure):
self._storage.save_retry_failure(retry.name, atom.name, last_failure)

61
taskflow/engines/action_engine/actions/task.py
View File

@@ -47,11 +47,13 @@ class TaskAction(base.Action):
return False
return True
def change_state(self, task, state,
progress=None, result=base.Action.NO_RESULT):
def change_state(
self, task, state, progress=None, result=base.Action.NO_RESULT
):
old_state = self._storage.get_atom_state(task.name)
if self._is_identity_transition(old_state, state, task,
progress=progress):
if self._is_identity_transition(
old_state, state, task, progress=progress
):
# NOTE(imelnikov): ignore identity transitions in order
# to avoid extra write to storage backend and, what's
# more important, extra notifications.
@@ -85,60 +87,71 @@ class TaskAction(base.Action):
pass
else:
try:
self._storage.set_task_progress(task.name, progress,
details=details)
self._storage.set_task_progress(
task.name, progress, details=details
)
except Exception:
# Update progress callbacks should never fail, so capture and
# log the emitted exception instead of raising it.
LOG.exception("Failed setting task progress for %s to %0.3f",
task, progress)
LOG.exception(
"Failed setting task progress for %s to %0.3f",
task,
progress,
)
def schedule_execution(self, task):
self.change_state(task, states.RUNNING, progress=0.0)
arguments = self._storage.fetch_mapped_args(
task.rebind,
atom_name=task.name,
optional_args=task.optional
task.rebind, atom_name=task.name, optional_args=task.optional
)
if task.notifier.can_be_registered(task_atom.EVENT_UPDATE_PROGRESS):
progress_callback = functools.partial(self._on_update_progress,
task)
progress_callback = functools.partial(
self._on_update_progress, task
)
else:
progress_callback = None
task_uuid = self._storage.get_atom_uuid(task.name)
return self._task_executor.execute_task(
task, task_uuid, arguments,
progress_callback=progress_callback)
task, task_uuid, arguments, progress_callback=progress_callback
)
def complete_execution(self, task, result):
if isinstance(result, failure.Failure):
self.change_state(task, states.FAILURE, result=result)
else:
self.change_state(task, states.SUCCESS,
result=result, progress=1.0)
self.change_state(
task, states.SUCCESS, result=result, progress=1.0
)
def schedule_reversion(self, task):
self.change_state(task, states.REVERTING, progress=0.0)
arguments = self._storage.fetch_mapped_args(
task.revert_rebind,
atom_name=task.name,
optional_args=task.revert_optional
optional_args=task.revert_optional,
)
task_uuid = self._storage.get_atom_uuid(task.name)
task_result = self._storage.get(task.name)
failures = self._storage.get_failures()
if task.notifier.can_be_registered(task_atom.EVENT_UPDATE_PROGRESS):
progress_callback = functools.partial(self._on_update_progress,
task)
progress_callback = functools.partial(
self._on_update_progress, task
)
else:
progress_callback = None
return self._task_executor.revert_task(
task, task_uuid, arguments, task_result, failures,
progress_callback=progress_callback)
task,
task_uuid,
arguments,
task_result,
failures,
progress_callback=progress_callback,
)
def complete_reversion(self, task, result):
if isinstance(result, failure.Failure):
self.change_state(task, states.REVERT_FAILURE, result=result)
else:
self.change_state(task, states.REVERTED, progress=1.0,
result=result)
self.change_state(
task, states.REVERTED, progress=1.0, result=result
)

76
taskflow/engines/action_engine/builder.py
View File

@@ -143,9 +143,12 @@ class MachineBuilder:
def do_schedule(next_nodes):
with self._storage.lock.write_lock():
return self._scheduler.schedule(
sorted(next_nodes,
key=lambda node: getattr(node, 'priority', 0),
reverse=True))
sorted(
next_nodes,
key=lambda node: getattr(node, 'priority', 0),
reverse=True,
)
)
def iter_next_atoms(atom=None, apply_deciders=True):
# Yields and filters and tweaks the next atoms to run...
@@ -165,8 +168,10 @@ class MachineBuilder:
# that are now ready to be ran.
with self._storage.lock.write_lock():
memory.next_up.update(
iter_utils.unique_seen((self._completer.resume(),
iter_next_atoms())))
iter_utils.unique_seen(
(self._completer.resume(), iter_next_atoms())
)
)
return SCHEDULE
def game_over(old_state, new_state, event):
@@ -181,13 +186,17 @@ class MachineBuilder:
# Avoid activating the deciders, since at this point
# the engine is finishing and there will be no more further
# work done anyway...
iter_next_atoms(apply_deciders=False))
iter_next_atoms(apply_deciders=False)
)
if leftover_atoms:
# Ok we didn't finish (either reverting or executing...) so
# that means we must of been stopped at some point...
LOG.trace("Suspension determined to have been reacted to"
" since (at least) %s atoms have been left in an"
" unfinished state", leftover_atoms)
LOG.trace(
"Suspension determined to have been reacted to"
" since (at least) %s atoms have been left in an"
" unfinished state",
leftover_atoms,
)
return SUSPENDED
elif self._runtime.is_success():
return SUCCESS
@@ -239,11 +248,16 @@ class MachineBuilder:
# would suck...)
if LOG.isEnabledFor(logging.DEBUG):
intention = get_atom_intention(atom.name)
LOG.debug("Discarding failure '%s' (in response"
" to outcome '%s') under completion"
" units request during completion of"
" atom '%s' (intention is to %s)",
result, outcome, atom, intention)
LOG.debug(
"Discarding failure '%s' (in response"
" to outcome '%s') under completion"
" units request during completion of"
" atom '%s' (intention is to %s)",
result,
outcome,
atom,
intention,
)
if gather_statistics:
statistics['discarded_failures'] += 1
if gather_statistics:
@@ -256,8 +270,7 @@ class MachineBuilder:
return WAS_CANCELLED
except Exception:
memory.failures.append(failure.Failure())
LOG.exception("Engine '%s' atom post-completion"
" failed", atom)
LOG.exception("Engine '%s' atom post-completion failed", atom)
return FAILED_COMPLETING
else:
return SUCCESSFULLY_COMPLETED
@@ -286,8 +299,10 @@ class MachineBuilder:
# before we iterate over any successors or predecessors
# that we know it has been completed and saved and so on...
completion_status = complete_an_atom(fut)
if (not memory.failures
and completion_status != WAS_CANCELLED):
if (
not memory.failures
and completion_status != WAS_CANCELLED
):
atom = fut.atom
try:
more_work = set(iter_next_atoms(atom=atom))
@@ -295,12 +310,17 @@ class MachineBuilder:
memory.failures.append(failure.Failure())
LOG.exception(
"Engine '%s' atom post-completion"
" next atom searching failed", atom)
" next atom searching failed",
atom,
)
else:
next_up.update(more_work)
current_flow_state = self._storage.get_flow_state()
if (current_flow_state == st.RUNNING
and next_up and not memory.failures):
if (
current_flow_state == st.RUNNING
and next_up
and not memory.failures
):
memory.next_up.update(next_up)
return SCHEDULE
elif memory.not_done:
@@ -311,8 +331,11 @@ class MachineBuilder:
return FINISH
def on_exit(old_state, event):
LOG.trace("Exiting old state '%s' in response to event '%s'",
old_state, event)
LOG.trace(
"Exiting old state '%s' in response to event '%s'",
old_state,
event,
)
if gather_statistics:
if old_state in watches:
w = watches[old_state]
@@ -324,8 +347,11 @@ class MachineBuilder:
statistics['awaiting'] = len(memory.next_up)
def on_enter(new_state, event):
LOG.trace("Entering new state '%s' in response to event '%s'",
new_state, event)
LOG.trace(
"Entering new state '%s' in response to event '%s'",
new_state,
event,
)
if gather_statistics and new_state in watches:
watches[new_state].restart()

73
taskflow/engines/action_engine/compiler.py
View File

@@ -25,7 +25,7 @@ from taskflow.types import tree as tr
from taskflow.utils import iter_utils
from taskflow.utils import misc
from taskflow.flow import (LINK_INVARIANT, LINK_RETRY) # noqa
from taskflow.flow import LINK_INVARIANT, LINK_RETRY # noqa
LOG = logging.getLogger(__name__)
@@ -105,8 +105,9 @@ class Compilation:
def _overlap_occurrence_detector(to_graph, from_graph):
"""Returns how many nodes in 'from' graph are in 'to' graph (if any)."""
return iter_utils.count(node for node in from_graph.nodes
if node in to_graph)
return iter_utils.count(
node for node in from_graph.nodes if node in to_graph
)
def _add_update_edges(graph, nodes_from, nodes_to, attr_dict=None):
@@ -162,22 +163,30 @@ class FlowCompiler:
tree_node.add(tr.Node(flow.retry, kind=RETRY))
decomposed = {
child: self._deep_compiler_func(child, parent=tree_node)[0]
for child in flow}
for child in flow
}
decomposed_graphs = list(decomposed.values())
graph = gr.merge_graphs(graph, *decomposed_graphs,
overlap_detector=_overlap_occurrence_detector)
graph = gr.merge_graphs(
graph,
*decomposed_graphs,
overlap_detector=_overlap_occurrence_detector,
)
for u, v, attr_dict in flow.iter_links():
u_graph = decomposed[u]
v_graph = decomposed[v]
_add_update_edges(graph, u_graph.no_successors_iter(),
list(v_graph.no_predecessors_iter()),
attr_dict=attr_dict)
_add_update_edges(
graph,
u_graph.no_successors_iter(),
list(v_graph.no_predecessors_iter()),
attr_dict=attr_dict,
)
# Insert the flow(s) retry if needed, and always make sure it
# is the **immediate** successor of the flow node itself.
if flow.retry is not None:
graph.add_node(flow.retry, kind=RETRY)
_add_update_edges(graph, [flow], [flow.retry],
attr_dict={LINK_INVARIANT: True})
_add_update_edges(
graph, [flow], [flow.retry], attr_dict={LINK_INVARIANT: True}
)
for node in graph.nodes:
if node is not flow.retry and node is not flow:
graph.nodes[node].setdefault(RETRY, flow.retry)
@@ -192,10 +201,16 @@ class FlowCompiler:
# us to easily know when we have entered a flow (when running) and
# do special and/or smart things such as only traverse up to the
# start of a flow when looking for node deciders.
_add_update_edges(graph, from_nodes, [
node for node in graph.no_predecessors_iter()
if node is not flow
], attr_dict=attr_dict)
_add_update_edges(
graph,
from_nodes,
[
node
for node in graph.no_predecessors_iter()
if node is not flow
],
attr_dict=attr_dict,
)
# Connect all nodes with no successors into a special terminator
# that is used to identify the end of the flow and ensure that all
# execution traversals will traverse over this node before executing
@@ -214,10 +229,16 @@ class FlowCompiler:
# that networkx provides??
flow_term = Terminator(flow)
graph.add_node(flow_term, kind=FLOW_END, noop=True)
_add_update_edges(graph, [
node for node in graph.no_successors_iter()
if node is not flow_term
], [flow_term], attr_dict={LINK_INVARIANT: True})
_add_update_edges(
graph,
[
node
for node in graph.no_successors_iter()
if node is not flow_term
],
[flow_term],
attr_dict={LINK_INVARIANT: True},
)
return graph, tree_node
@@ -337,15 +358,19 @@ class PatternCompiler:
self._post_item_compile(item, graph, node)
return graph, node
else:
raise TypeError("Unknown object '%s' (%s) requested to compile"
% (item, type(item)))
raise TypeError(
"Unknown object '%s' (%s) requested to compile"
% (item, type(item))
)
def _pre_item_compile(self, item):
"""Called before a item is compiled; any pre-compilation actions."""
if item in self._history:
raise ValueError("Already compiled item '%s' (%s), duplicate"
" and/or recursive compiling is not"
" supported" % (item, type(item)))
raise ValueError(
"Already compiled item '%s' (%s), duplicate"
" and/or recursive compiling is not"
" supported" % (item, type(item))
)
self._history.add(item)
if LOG.isEnabledFor(logging.TRACE):
LOG.trace("%sCompiling '%s'", " " * self._level, item)

77
taskflow/engines/action_engine/completer.py
View File

@@ -58,10 +58,14 @@ class RevertAndRetry(Strategy):
self._retry = retry
def apply(self):
tweaked = self._runtime.reset_atoms([self._retry], state=None,
intention=st.RETRY)
tweaked.extend(self._runtime.reset_subgraph(self._retry, state=None,
intention=st.REVERT))
tweaked = self._runtime.reset_atoms(
[self._retry], state=None, intention=st.RETRY
)
tweaked.extend(
self._runtime.reset_subgraph(
self._retry, state=None, intention=st.REVERT
)
)
return tweaked
@@ -76,7 +80,9 @@ class RevertAll(Strategy):
def apply(self):
return self._runtime.reset_atoms(
self._runtime.iterate_nodes(co.ATOMS),
state=None, intention=st.REVERT)
state=None,
intention=st.REVERT,
)
class Revert(Strategy):
@@ -89,10 +95,14 @@ class Revert(Strategy):
self._atom = atom
def apply(self):
tweaked = self._runtime.reset_atoms([self._atom], state=None,
intention=st.REVERT)
tweaked.extend(self._runtime.reset_subgraph(self._atom, state=None,
intention=st.REVERT))
tweaked = self._runtime.reset_atoms(
[self._atom], state=None, intention=st.REVERT
)
tweaked.extend(
self._runtime.reset_subgraph(
self._atom, state=None, intention=st.REVERT
)
)
return tweaked
@@ -104,9 +114,11 @@ class Completer:
self._storage = runtime.storage
self._undefined_resolver = RevertAll(self._runtime)
self._defer_reverts = strutils.bool_from_string(
self._runtime.options.get('defer_reverts', False))
self._runtime.options.get('defer_reverts', False)
)
self._resolve = not strutils.bool_from_string(
self._runtime.options.get('never_resolve', False))
self._runtime.options.get('never_resolve', False)
)
def resume(self):
"""Resumes atoms in the contained graph.
@@ -120,14 +132,16 @@ class Completer:
attempt not previously finishing).
"""
atoms = list(self._runtime.iterate_nodes(co.ATOMS))
atom_states = self._storage.get_atoms_states(atom.name
for atom in atoms)
atom_states = self._storage.get_atoms_states(
atom.name for atom in atoms
)
if self._resolve:
for atom in atoms:
atom_state, _atom_intention = atom_states[atom.name]
if atom_state == st.FAILURE:
self._process_atom_failure(
atom, self._storage.get(atom.name))
atom, self._storage.get(atom.name)
)
for retry in self._runtime.iterate_retries(st.RETRYING):
retry_affected_atoms_it = self._runtime.retry_subflow(retry)
for atom, state, intention in retry_affected_atoms_it:
@@ -138,8 +152,11 @@ class Completer:
atom_state, _atom_intention = atom_states[atom.name]
if atom_state in (st.RUNNING, st.REVERTING):
unfinished_atoms.add(atom)
LOG.trace("Resuming atom '%s' since it was left in"
" state %s", atom, atom_state)
LOG.trace(
"Resuming atom '%s' since it was left in state %s",
atom,
atom_state,
)
return unfinished_atoms
def complete_failure(self, node, outcome, failure):
@@ -192,8 +209,10 @@ class Completer:
elif strategy == retry_atom.REVERT_ALL:
return RevertAll(self._runtime)
else:
raise ValueError("Unknown atom failure resolution"
" action/strategy '%s'" % strategy)
raise ValueError(
"Unknown atom failure resolution"
" action/strategy '%s'" % strategy
)
else:
return self._undefined_resolver
@@ -207,14 +226,24 @@ class Completer:
the failure can be worked around.
"""
resolver = self._determine_resolution(atom, failure)
LOG.debug("Applying resolver '%s' to resolve failure '%s'"
" of atom '%s'", resolver, failure, atom)
LOG.debug(
"Applying resolver '%s' to resolve failure '%s' of atom '%s'",
resolver,
failure,
atom,
)
tweaked = resolver.apply()
# Only show the tweaked node list when trace is on, otherwise
# just show the amount/count of nodes tweaks...
if LOG.isEnabledFor(logging.TRACE):
LOG.trace("Modified/tweaked %s nodes while applying"
" resolver '%s'", tweaked, resolver)
LOG.trace(
"Modified/tweaked %s nodes while applying resolver '%s'",
tweaked,
resolver,
)
else:
LOG.debug("Modified/tweaked %s nodes while applying"
" resolver '%s'", len(tweaked), resolver)
LOG.debug(
"Modified/tweaked %s nodes while applying resolver '%s'",
len(tweaked),
resolver,
)

72
taskflow/engines/action_engine/deciders.py
View File

@@ -67,23 +67,34 @@ class Decider(metaclass=abc.ABCMeta):
def _affect_all_successors(atom, runtime):
execution_graph = runtime.compilation.execution_graph
successors_iter = traversal.depth_first_iterate(
execution_graph, atom, traversal.Direction.FORWARD)
runtime.reset_atoms(itertools.chain([atom], successors_iter),
state=states.IGNORE, intention=states.IGNORE)
execution_graph, atom, traversal.Direction.FORWARD
)
runtime.reset_atoms(
itertools.chain([atom], successors_iter),
state=states.IGNORE,
intention=states.IGNORE,
)
def _affect_successor_tasks_in_same_flow(atom, runtime):
execution_graph = runtime.compilation.execution_graph
successors_iter = traversal.depth_first_iterate(
execution_graph, atom, traversal.Direction.FORWARD,
execution_graph,
atom,
traversal.Direction.FORWARD,
# Do not go through nested flows but do follow *all* tasks that
# are directly connected in this same flow (thus the reason this is
# called the same flow decider); retries are direct successors
# of flows, so they should also be not traversed through, but
# setting this explicitly ensures that.
through_flows=False, through_retries=False)
runtime.reset_atoms(itertools.chain([atom], successors_iter),
state=states.IGNORE, intention=states.IGNORE)
through_flows=False,
through_retries=False,
)
runtime.reset_atoms(
itertools.chain([atom], successors_iter),
state=states.IGNORE,
intention=states.IGNORE,
)
def _affect_atom(atom, runtime):
@@ -97,9 +108,13 @@ def _affect_direct_task_neighbors(atom, runtime):
node_data = execution_graph.nodes[node]
if node_data['kind'] == compiler.TASK:
yield node
successors_iter = _walk_neighbors()
runtime.reset_atoms(itertools.chain([atom], successors_iter),
state=states.IGNORE, intention=states.IGNORE)
runtime.reset_atoms(
itertools.chain([atom], successors_iter),
state=states.IGNORE,
intention=states.IGNORE,
)
class IgnoreDecider(Decider):
@@ -128,18 +143,23 @@ class IgnoreDecider(Decider):
# that those results can be used by the decider(s) that are
# making a decision as to pass or not pass...
states_intentions = runtime.storage.get_atoms_states(
ed.from_node.name for ed in self._edge_deciders
if ed.kind in compiler.ATOMS)
ed.from_node.name
for ed in self._edge_deciders
if ed.kind in compiler.ATOMS
)
for atom_name in states_intentions.keys():
atom_state, _atom_intention = states_intentions[atom_name]
if atom_state != states.IGNORE:
history[atom_name] = runtime.storage.get(atom_name)
for ed in self._edge_deciders:
if (ed.kind in compiler.ATOMS and
# It was an ignored atom (not included in history and
# the only way that is possible is via above loop
# skipping it...)
ed.from_node.name not in history):
if (
ed.kind in compiler.ATOMS
and
# It was an ignored atom (not included in history and
# the only way that is possible is via above loop
# skipping it...)
ed.from_node.name not in history
):
voters['ignored'].append(ed)
continue
if not ed.decider(history=history):
@@ -147,15 +167,17 @@ class IgnoreDecider(Decider):
else:
voters['run_it'].append(ed)
if LOG.isEnabledFor(logging.TRACE):
LOG.trace("Out of %s deciders there were %s 'do no run it'"
" voters, %s 'do run it' voters and %s 'ignored'"
" voters for transition to atom '%s' given history %s",
sum(len(eds) for eds in voters.values()),
list(ed.from_node.name
for ed in voters['do_not_run_it']),
list(ed.from_node.name for ed in voters['run_it']),
list(ed.from_node.name for ed in voters['ignored']),
self._atom.name, history)
LOG.trace(
"Out of %s deciders there were %s 'do no run it'"
" voters, %s 'do run it' voters and %s 'ignored'"
" voters for transition to atom '%s' given history %s",
sum(len(eds) for eds in voters.values()),
list(ed.from_node.name for ed in voters['do_not_run_it']),
list(ed.from_node.name for ed in voters['run_it']),
list(ed.from_node.name for ed in voters['ignored']),
self._atom.name,
history,
)
return voters['do_not_run_it']
def affect(self, runtime, nay_voters):

289
taskflow/engines/action_engine/engine.py
View File

@@ -55,8 +55,9 @@ def _start_stop(task_executor, retry_executor):
task_executor.stop()
def _pre_check(check_compiled=True, check_storage_ensured=True,
check_validated=True):
def _pre_check(
check_compiled=True, check_storage_ensured=True, check_validated=True
):
"""Engine state precondition checking decorator."""
def decorator(meth):
@@ -65,15 +66,21 @@ def _pre_check(check_compiled=True, check_storage_ensured=True,
@functools.wraps(meth)
def wrapper(self, *args, **kwargs):
if check_compiled and not self._compiled:
raise exc.InvalidState("Can not %s an engine which"
" has not been compiled" % do_what)
raise exc.InvalidState(
"Can not %s an engine which"
" has not been compiled" % do_what
)
if check_storage_ensured and not self._storage_ensured:
raise exc.InvalidState("Can not %s an engine"
" which has not had its storage"
" populated" % do_what)
raise exc.InvalidState(
"Can not %s an engine"
" which has not had its storage"
" populated" % do_what
)
if check_validated and not self._validated:
raise exc.InvalidState("Can not %s an engine which"
" has not been validated" % do_what)
raise exc.InvalidState(
"Can not %s an engine which"
" has not been validated" % do_what
)
return meth(self, *args, **kwargs)
return wrapper
@@ -148,8 +155,16 @@ class ActionEngine(base.Engine):
"""
IGNORABLE_STATES = frozenset(
itertools.chain([states.SCHEDULING, states.WAITING, states.RESUMING,
states.ANALYZING], builder.META_STATES))
itertools.chain(
[
states.SCHEDULING,
states.WAITING,
states.RESUMING,
states.ANALYZING,
],
builder.META_STATES,
)
)
"""
Informational states this engines internal machine yields back while
running, not useful to have the engine record but useful to provide to
@@ -175,18 +190,22 @@ class ActionEngine(base.Engine):
# or thread (this could change in the future if we desire it to).
self._retry_executor = executor.SerialRetryExecutor()
self._inject_transient = strutils.bool_from_string(
self._options.get('inject_transient', True))
self._options.get('inject_transient', True)
)
self._gather_statistics = strutils.bool_from_string(
self._options.get('gather_statistics', True))
self._options.get('gather_statistics', True)
)
self._statistics = {}
@_pre_check(check_compiled=True,
# NOTE(harlowja): We can alter the state of the
# flow without ensuring its storage is setup for
# its atoms (since this state change does not affect
# those units).
check_storage_ensured=False,
check_validated=False)
@_pre_check(
check_compiled=True,
# NOTE(harlowja): We can alter the state of the
# flow without ensuring its storage is setup for
# its atoms (since this state change does not affect
# those units).
check_storage_ensured=False,
check_validated=False,
)
def suspend(self):
self._change_state(states.SUSPENDING)
@@ -221,14 +240,18 @@ class ActionEngine(base.Engine):
the actual runtime lookup strategy, which typically will be, but is
not always different).
"""
def _scope_fetcher(atom_name):
if self._compiled:
return self._runtime.fetch_scopes_for(atom_name)
else:
return None
return storage.Storage(self._flow_detail,
backend=self._backend,
scope_fetcher=_scope_fetcher)
return storage.Storage(
self._flow_detail,
backend=self._backend,
scope_fetcher=_scope_fetcher,
)
def run(self, timeout=None):
"""Runs the engine (or die trying).
@@ -239,8 +262,9 @@ class ActionEngine(base.Engine):
"""
with fasteners.try_lock(self._lock) as was_locked:
if not was_locked:
raise exc.ExecutionFailure("Engine currently locked, please"
" try again later")
raise exc.ExecutionFailure(
"Engine currently locked, please try again later"
)
for _state in self.run_iter(timeout=timeout):
pass
@@ -272,7 +296,8 @@ class ActionEngine(base.Engine):
# are quite useful to log (and the performance of tracking this
# should be negligible).
last_transitions = collections.deque(
maxlen=max(1, self.MAX_MACHINE_STATES_RETAINED))
maxlen=max(1, self.MAX_MACHINE_STATES_RETAINED)
)
with _start_stop(self._task_executor, self._retry_executor):
self._change_state(states.RUNNING)
if self._gather_statistics:
@@ -284,8 +309,10 @@ class ActionEngine(base.Engine):
try:
closed = False
machine, memory = self._runtime.builder.build(
self._statistics, timeout=timeout,
gather_statistics=self._gather_statistics)
self._statistics,
timeout=timeout,
gather_statistics=self._gather_statistics,
)
r = runners.FiniteRunner(machine)
for transition in r.run_iter(builder.START):
last_transitions.append(transition)
@@ -317,11 +344,13 @@ class ActionEngine(base.Engine):
self.suspend()
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception("Engine execution has failed, something"
" bad must have happened (last"
" %s machine transitions were %s)",
last_transitions.maxlen,
list(last_transitions))
LOG.exception(
"Engine execution has failed, something"
" bad must have happened (last"
" %s machine transitions were %s)",
last_transitions.maxlen,
list(last_transitions),
)
self._change_state(states.FAILURE)
else:
if last_transitions:
@@ -332,8 +361,8 @@ class ActionEngine(base.Engine):
e_failures = self.storage.get_execute_failures()
r_failures = self.storage.get_revert_failures()
er_failures = itertools.chain(
e_failures.values(),
r_failures.values())
e_failures.values(), r_failures.values()
)
failure.Failure.reraise_if_any(er_failures)
finally:
if w is not None:
@@ -355,7 +384,8 @@ class ActionEngine(base.Engine):
seen.add(atom_name)
if dups:
raise exc.Duplicate(
"Atoms with duplicate names found: %s" % (sorted(dups)))
"Atoms with duplicate names found: %s" % (sorted(dups))
)
return compilation
def _change_state(self, state):
@@ -371,12 +401,12 @@ class ActionEngine(base.Engine):
def _ensure_storage(self):
"""Ensure all contained atoms exist in the storage unit."""
self.storage.ensure_atoms(
self._runtime.iterate_nodes(compiler.ATOMS))
self.storage.ensure_atoms(self._runtime.iterate_nodes(compiler.ATOMS))
for atom in self._runtime.iterate_nodes(compiler.ATOMS):
if atom.inject:
self.storage.inject_atom_args(atom.name, atom.inject,
transient=self._inject_transient)
self.storage.inject_atom_args(
atom.name, atom.inject, transient=self._inject_transient
)
@fasteners.locked
@_pre_check(check_validated=False)
@@ -387,11 +417,14 @@ class ActionEngine(base.Engine):
# by failing at validation time).
if LOG.isEnabledFor(logging.TRACE):
execution_graph = self._compilation.execution_graph
LOG.trace("Validating scoping and argument visibility for"
" execution graph with %s nodes and %s edges with"
" density %0.3f", execution_graph.number_of_nodes(),
execution_graph.number_of_edges(),
nx.density(execution_graph))
LOG.trace(
"Validating scoping and argument visibility for"
" execution graph with %s nodes and %s edges with"
" density %0.3f",
execution_graph.number_of_nodes(),
execution_graph.number_of_edges(),
nx.density(execution_graph),
)
missing = set()
# Attempt to retain a chain of what was missing (so that the final
# raised exception for the flow has the nodes that had missing
@@ -401,18 +434,25 @@ class ActionEngine(base.Engine):
missing_nodes = 0
for atom in self._runtime.iterate_nodes(compiler.ATOMS):
exec_missing = self.storage.fetch_unsatisfied_args(
atom.name, atom.rebind, optional_args=atom.optional)
atom.name, atom.rebind, optional_args=atom.optional
)
revert_missing = self.storage.fetch_unsatisfied_args(
atom.name, atom.revert_rebind,
optional_args=atom.revert_optional)
atom_missing = (('execute', exec_missing),
('revert', revert_missing))
atom.name,
atom.revert_rebind,
optional_args=atom.revert_optional,
)
atom_missing = (
('execute', exec_missing),
('revert', revert_missing),
)
for method, method_missing in atom_missing:
if method_missing:
cause = exc.MissingDependencies(atom,
sorted(method_missing),
cause=last_cause,
method=method)
cause = exc.MissingDependencies(
atom,
sorted(method_missing),
cause=last_cause,
method=method,
)
last_cause = cause
last_node = atom
missing_nodes += 1
@@ -424,9 +464,9 @@ class ActionEngine(base.Engine):
if missing_nodes == 1 and last_node is self._flow:
raise last_cause
else:
raise exc.MissingDependencies(self._flow,
sorted(missing),
cause=last_cause)
raise exc.MissingDependencies(
self._flow, sorted(missing), cause=last_cause
)
self._validated = True
@fasteners.locked
@@ -458,12 +498,14 @@ class ActionEngine(base.Engine):
if self._compiled:
return
self._compilation = self._check_compilation(self._compiler.compile())
self._runtime = runtime.Runtime(self._compilation,
self.storage,
self.atom_notifier,
self._task_executor,
self._retry_executor,
options=self._options)
self._runtime = runtime.Runtime(
self._compilation,
self.storage,
self.atom_notifier,
self._task_executor,
self._retry_executor,
options=self._options,
)
self._runtime.compile()
self._compiled = True
@@ -476,14 +518,16 @@ class SerialActionEngine(ActionEngine):
self._task_executor = executor.SerialTaskExecutor()
class _ExecutorTypeMatch(collections.namedtuple('_ExecutorTypeMatch',
['types', 'executor_cls'])):
class _ExecutorTypeMatch(
collections.namedtuple('_ExecutorTypeMatch', ['types', 'executor_cls'])
):
def matches(self, executor):
return isinstance(executor, self.types)
class _ExecutorTextMatch(collections.namedtuple('_ExecutorTextMatch',
['strings', 'executor_cls'])):
class _ExecutorTextMatch(
collections.namedtuple('_ExecutorTextMatch', ['strings', 'executor_cls'])
):
def matches(self, text):
return text.lower() in self.strings
@@ -494,51 +538,52 @@ class ParallelActionEngine(ActionEngine):
**Additional engine options:**
* ``executor``: a object that implements a :pep:`3148` compatible executor
interface; it will be used for scheduling tasks. The following
type are applicable (other unknown types passed will cause a type
error to be raised).
interface; it will be used for scheduling tasks. The following type are
applicable (other unknown types passed will cause a type error to be
raised).
========================= ===============================================
Type provided Executor used
========================= ===============================================
|cft|.ThreadPoolExecutor :class:`~.executor.ParallelThreadTaskExecutor`
|cfp|.ProcessPoolExecutor :class:`~.|pe|.ParallelProcessTaskExecutor`
|cf|._base.Executor :class:`~.executor.ParallelThreadTaskExecutor`
========================= ===============================================
========================= ==============================================
Type provided Executor used
========================= ==============================================
|cft|.ThreadPoolExecutor :class:`~.executor.ParallelThreadTaskExecutor`
|cfp|.ProcessPoolExecutor :class:`~.|pe|.ParallelProcessTaskExecutor`
|cf|._base.Executor :class:`~.executor.ParallelThreadTaskExecutor`
========================= ==============================================
* ``executor``: a string that will be used to select a :pep:`3148`
compatible executor; it will be used for scheduling tasks. The following
string are applicable (other unknown strings passed will cause a value
error to be raised).
=========================== ===============================================
String (case insensitive) Executor used
=========================== ===============================================
``process`` :class:`~.|pe|.ParallelProcessTaskExecutor`
``processes`` :class:`~.|pe|.ParallelProcessTaskExecutor`
``thread`` :class:`~.executor.ParallelThreadTaskExecutor`
``threaded`` :class:`~.executor.ParallelThreadTaskExecutor`
``threads`` :class:`~.executor.ParallelThreadTaskExecutor`
``greenthread`` :class:`~.executor.ParallelThreadTaskExecutor`
(greened version)
``greedthreaded`` :class:`~.executor.ParallelThreadTaskExecutor`
(greened version)
``greenthreads`` :class:`~.executor.ParallelThreadTaskExecutor`
(greened version)
=========================== ===============================================
=========================== ==============================================
String (case insensitive) Executor used
=========================== ==============================================
``process`` :class:`~.|pe|.ParallelProcessTaskExecutor`
``processes`` :class:`~.|pe|.ParallelProcessTaskExecutor`
``thread`` :class:`~.executor.ParallelThreadTaskExecutor`
``threaded`` :class:`~.executor.ParallelThreadTaskExecutor`
``threads`` :class:`~.executor.ParallelThreadTaskExecutor`
``greenthread`` :class:`~.executor.ParallelThreadTaskExecutor`
(greened version)
``greedthreaded`` :class:`~.executor.ParallelThreadTaskExecutor`
(greened version)
``greenthreads`` :class:`~.executor.ParallelThreadTaskExecutor`
(greened version)
=========================== ==============================================
* ``max_workers``: a integer that will affect the number of parallel
workers that are used to dispatch tasks into (this number is bounded
by the maximum parallelization your workflow can support).
* ``wait_timeout``: a float (in seconds) that will affect the
parallel process task executor (and therefore is **only** applicable when
the executor provided above is of the process variant). This number
affects how much time the process task executor waits for messages from
child processes (typically indicating they have finished or failed). A
lower number will have high granularity but *currently* involves more
polling while a higher number will involve less polling but a slower time
for an engine to notice a task has completed.
parallel process task executor (and therefore is **only** applicable
when the executor provided above is of the process variant). This
number affects how much time the process task executor waits for
messages from child processes (typically indicating they have
finished or failed). A lower number will have high granularity but
*currently* involves more polling while a higher number will involve
less polling but a slower time for an engine to notice a task has
completed.
.. |cfp| replace:: concurrent.futures.process
.. |cft| replace:: concurrent.futures.thread
@@ -552,21 +597,26 @@ String (case insensitive) Executor used
# allow for instances of that to be detected and handled correctly, instead
# of forcing everyone to use our derivatives (futurist or other)...
_executor_cls_matchers = [
_ExecutorTypeMatch((futures.ThreadPoolExecutor,),
executor.ParallelThreadTaskExecutor),
_ExecutorTypeMatch((futures.Executor,),
executor.ParallelThreadTaskExecutor),
_ExecutorTypeMatch(
(futures.ThreadPoolExecutor,), executor.ParallelThreadTaskExecutor
),
_ExecutorTypeMatch(
(futures.Executor,), executor.ParallelThreadTaskExecutor
),
]
# One of these should match when a string/text is provided for the
# 'executor' option (a mixed case equivalent is allowed since the match
# will be lower-cased before checking).
_executor_str_matchers = [
_ExecutorTextMatch(frozenset(['thread', 'threads', 'threaded']),
executor.ParallelThreadTaskExecutor),
_ExecutorTextMatch(frozenset(['greenthread', 'greenthreads',
'greenthreaded']),
executor.ParallelGreenThreadTaskExecutor),
_ExecutorTextMatch(
frozenset(['thread', 'threads', 'threaded']),
executor.ParallelThreadTaskExecutor,
),
_ExecutorTextMatch(
frozenset(['greenthread', 'greenthreads', 'greenthreaded']),
executor.ParallelGreenThreadTaskExecutor,
),
]
# Used when no executor is provided (either a string or object)...
@@ -594,9 +644,11 @@ String (case insensitive) Executor used
expected = set()
for m in cls._executor_str_matchers:
expected.update(m.strings)
raise ValueError("Unknown executor string '%s' expected"
" one of %s (or mixed case equivalent)"
% (desired_executor, list(expected)))
raise ValueError(
"Unknown executor string '%s' expected"
" one of %s (or mixed case equivalent)"
% (desired_executor, list(expected))
)
else:
executor_cls = matched_executor_cls
elif desired_executor is not None:
@@ -609,15 +661,20 @@ String (case insensitive) Executor used
expected = set()
for m in cls._executor_cls_matchers:
expected.update(m.types)
raise TypeError("Unknown executor '%s' (%s) expected an"
" instance of %s" % (desired_executor,
type(desired_executor),
list(expected)))
raise TypeError(
"Unknown executor '%s' (%s) expected an"
" instance of %s"
% (
desired_executor,
type(desired_executor),
list(expected),
)
)
else:
executor_cls = matched_executor_cls
kwargs['executor'] = desired_executor
try:
for (k, value_converter) in executor_cls.constructor_options:
for k, value_converter in executor_cls.constructor_options:
try:
kwargs[k] = value_converter(options[k])
except KeyError:

83
taskflow/engines/action_engine/executor.py
View File

@@ -42,9 +42,9 @@ def _revert_retry(retry, arguments):
def _execute_task(task, arguments, progress_callback=None):
with notifier.register_deregister(task.notifier,
ta.EVENT_UPDATE_PROGRESS,
callback=progress_callback):
with notifier.register_deregister(
task.notifier, ta.EVENT_UPDATE_PROGRESS, callback=progress_callback
):
try:
task.pre_execute()
result = task.execute(**arguments)
@@ -61,9 +61,9 @@ def _revert_task(task, arguments, result, failures, progress_callback=None):
arguments = arguments.copy()
arguments[ta.REVERT_RESULT] = result
arguments[ta.REVERT_FLOW_FAILURES] = failures
with notifier.register_deregister(task.notifier,
ta.EVENT_UPDATE_PROGRESS,
callback=progress_callback):
with notifier.register_deregister(
task.notifier, ta.EVENT_UPDATE_PROGRESS, callback=progress_callback
):
try:
task.pre_revert()
result = task.revert(**arguments)
@@ -112,13 +112,19 @@ class TaskExecutor(metaclass=abc.ABCMeta):
"""
@abc.abstractmethod
def execute_task(self, task, task_uuid, arguments,
progress_callback=None):
def execute_task(self, task, task_uuid, arguments, progress_callback=None):
"""Schedules task execution."""
@abc.abstractmethod
def revert_task(self, task, task_uuid, arguments, result, failures,
progress_callback=None):
def revert_task(
self,
task,
task_uuid,
arguments,
result,
failures,
progress_callback=None,
):
"""Schedules task reversion."""
def start(self):
@@ -141,17 +147,29 @@ class SerialTaskExecutor(TaskExecutor):
self._executor.shutdown()
def execute_task(self, task, task_uuid, arguments, progress_callback=None):
fut = self._executor.submit(_execute_task,
task, arguments,
progress_callback=progress_callback)
fut = self._executor.submit(
_execute_task, task, arguments, progress_callback=progress_callback
)
fut.atom = task
return fut
def revert_task(self, task, task_uuid, arguments, result, failures,
progress_callback=None):
fut = self._executor.submit(_revert_task,
task, arguments, result, failures,
progress_callback=progress_callback)
def revert_task(
self,
task,
task_uuid,
arguments,
result,
failures,
progress_callback=None,
):
fut = self._executor.submit(
_revert_task,
task,
arguments,
result,
failures,
progress_callback=progress_callback,
)
fut.atom = task
return fut
@@ -188,18 +206,33 @@ class ParallelTaskExecutor(TaskExecutor):
return fut
def execute_task(self, task, task_uuid, arguments, progress_callback=None):
return self._submit_task(_execute_task, task, arguments,
progress_callback=progress_callback)
return self._submit_task(
_execute_task, task, arguments, progress_callback=progress_callback
)
def revert_task(self, task, task_uuid, arguments, result, failures,
progress_callback=None):
return self._submit_task(_revert_task, task, arguments, result,
failures, progress_callback=progress_callback)
def revert_task(
self,
task,
task_uuid,
arguments,
result,
failures,
progress_callback=None,
):
return self._submit_task(
_revert_task,
task,
arguments,
result,
failures,
progress_callback=progress_callback,
)
def start(self):
if self._own_executor:
self._executor = self._create_executor(
max_workers=self._max_workers)
max_workers=self._max_workers
)
def stop(self):
if self._own_executor:

89
taskflow/engines/action_engine/runtime.py
View File

@@ -32,11 +32,12 @@ from taskflow import logging
from taskflow import states as st
from taskflow.utils import misc
from taskflow.flow import (LINK_DECIDER, LINK_DECIDER_DEPTH) # noqa
from taskflow.flow import LINK_DECIDER, LINK_DECIDER_DEPTH # noqa
# Small helper to make the edge decider tuples more easily useable...
_EdgeDecider = collections.namedtuple('_EdgeDecider',
'from_node,kind,decider,depth')
_EdgeDecider = collections.namedtuple(
'_EdgeDecider', 'from_node,kind,decider,depth'
)
LOG = logging.getLogger(__name__)
@@ -49,9 +50,15 @@ class Runtime:
action engine to run to completion.
"""
def __init__(self, compilation, storage, atom_notifier,
task_executor, retry_executor,
options=None):
def __init__(
self,
compilation,
storage,
atom_notifier,
task_executor,
retry_executor,
options=None,
):
self._atom_notifier = atom_notifier
self._task_executor = task_executor
self._retry_executor = retry_executor
@@ -65,8 +72,9 @@ class Runtime:
# This is basically a reverse breadth first exploration, with
# special logic to further traverse down flow nodes as needed...
predecessors_iter = graph.predecessors
nodes = collections.deque((u_node, atom)
for u_node in predecessors_iter(atom))
nodes = collections.deque(
(u_node, atom) for u_node in predecessors_iter(atom)
)
visited = set()
while nodes:
u_node, v_node = nodes.popleft()
@@ -77,8 +85,7 @@ class Runtime:
decider_depth = u_v_data.get(LINK_DECIDER_DEPTH)
if decider_depth is None:
decider_depth = de.Depth.ALL
yield _EdgeDecider(u_node, u_node_kind,
decider, decider_depth)
yield _EdgeDecider(u_node, u_node_kind, decider, decider_depth)
except KeyError:
pass
if u_node_kind == com.FLOW and u_node not in visited:
@@ -89,8 +96,10 @@ class Runtime:
# sure that any prior decider that was directed at this flow
# node also gets used during future decisions about this
# atom node.
nodes.extend((u_u_node, u_node)
for u_u_node in predecessors_iter(u_node))
nodes.extend(
(u_u_node, u_node)
for u_u_node in predecessors_iter(u_node)
)
def compile(self):
"""Compiles & caches frequently used execution helper objects.
@@ -102,8 +111,9 @@ class Runtime:
specific scheduler and so-on).
"""
change_state_handlers = {
com.TASK: functools.partial(self.task_action.change_state,
progress=0.0),
com.TASK: functools.partial(
self.task_action.change_state, progress=0.0
),
com.RETRY: self.retry_action.change_state,
}
schedulers = {
@@ -129,8 +139,9 @@ class Runtime:
scheduler = schedulers[node_kind]
action = actions[node_kind]
else:
raise exc.CompilationFailure("Unknown node kind '%s'"
" encountered" % node_kind)
raise exc.CompilationFailure(
"Unknown node kind '%s' encountered" % node_kind
)
metadata = {}
deciders_it = self._walk_edge_deciders(graph, node)
walker = sc.ScopeWalker(self.compilation, node, names_only=True)
@@ -140,8 +151,12 @@ class Runtime:
metadata['scheduler'] = scheduler
metadata['edge_deciders'] = tuple(deciders_it)
metadata['action'] = action
LOG.trace("Compiled %s metadata for node %s (%s)",
metadata, node.name, node_kind)
LOG.trace(
"Compiled %s metadata for node %s (%s)",
metadata,
node.name,
node_kind,
)
self._atom_cache[node.name] = metadata
# TODO(harlowja): optimize the different decider depths to avoid
# repeated full successor searching; this can be done by searching
@@ -186,15 +201,15 @@ class Runtime:
@misc.cachedproperty
def retry_action(self):
return ra.RetryAction(self._storage,
self._atom_notifier,
self._retry_executor)
return ra.RetryAction(
self._storage, self._atom_notifier, self._retry_executor
)
@misc.cachedproperty
def task_action(self):
return ta.TaskAction(self._storage,
self._atom_notifier,
self._task_executor)
return ta.TaskAction(
self._storage, self._atom_notifier, self._task_executor
)
def _fetch_atom_metadata_entry(self, atom_name, metadata_key):
return self._atom_cache[atom_name][metadata_key]
@@ -205,7 +220,8 @@ class Runtime:
# internally to the engine, and is not exposed to atoms that will
# not exist and therefore doesn't need to handle that case).
check_transition_handler = self._fetch_atom_metadata_entry(
atom.name, 'check_transition_handler')
atom.name, 'check_transition_handler'
)
return check_transition_handler(current_state, target_state)
def fetch_edge_deciders(self, atom):
@@ -250,8 +266,9 @@ class Runtime:
"""
if state:
atoms = list(self.iterate_nodes((com.RETRY,)))
atom_states = self._storage.get_atoms_states(atom.name
for atom in atoms)
atom_states = self._storage.get_atoms_states(
atom.name for atom in atoms
)
for atom in atoms:
atom_state, _atom_intention = atom_states[atom.name]
if atom_state == state:
@@ -270,8 +287,9 @@ class Runtime:
def is_success(self):
"""Checks if all atoms in the execution graph are in 'happy' state."""
atoms = list(self.iterate_nodes(com.ATOMS))
atom_states = self._storage.get_atoms_states(atom.name
for atom in atoms)
atom_states = self._storage.get_atoms_states(
atom.name for atom in atoms
)
for atom in atoms:
atom_state, _atom_intention = atom_states[atom.name]
if atom_state == st.IGNORE:
@@ -305,7 +323,8 @@ class Runtime:
tweaked.append((atom, state, intention))
if state:
change_state_handler = self._fetch_atom_metadata_entry(
atom.name, 'change_state_handler')
atom.name, 'change_state_handler'
)
change_state_handler(atom, state)
if intention:
self.storage.set_atom_intention(atom.name, intention)
@@ -313,8 +332,9 @@ class Runtime:
def reset_all(self, state=st.PENDING, intention=st.EXECUTE):
"""Resets all atoms to the given state and intention."""
return self.reset_atoms(self.iterate_nodes(com.ATOMS),
state=state, intention=intention)
return self.reset_atoms(
self.iterate_nodes(com.ATOMS), state=state, intention=intention
)
def reset_subgraph(self, atom, state=st.PENDING, intention=st.EXECUTE):
"""Resets a atoms subgraph to the given state and intention.
@@ -322,8 +342,9 @@ class Runtime:
The subgraph is contained of **all** of the atoms successors.
"""
execution_graph = self._compilation.execution_graph
atoms_it = tr.depth_first_iterate(execution_graph, atom,
tr.Direction.FORWARD)
atoms_it = tr.depth_first_iterate(
execution_graph, atom, tr.Direction.FORWARD
)
return self.reset_atoms(atoms_it, state=state, intention=intention)
def retry_subflow(self, retry):

10
taskflow/engines/action_engine/scheduler.py
View File

@@ -46,8 +46,9 @@ class RetryScheduler:
self._runtime.retry_subflow(retry)
return self._retry_action.schedule_execution(retry)
else:
raise excp.ExecutionFailure("Unknown how to schedule retry with"
" intention: %s" % intention)
raise excp.ExecutionFailure(
"Unknown how to schedule retry with intention: %s" % intention
)
class TaskScheduler:
@@ -69,8 +70,9 @@ class TaskScheduler:
elif intention == st.REVERT:
return self._task_action.schedule_reversion(task)
else:
raise excp.ExecutionFailure("Unknown how to schedule task with"
" intention: %s" % intention)
raise excp.ExecutionFailure(
"Unknown how to schedule task with intention: %s" % intention
)
class Scheduler:

25
taskflow/engines/action_engine/scopes.py
View File

@@ -32,8 +32,9 @@ class ScopeWalker:
def __init__(self, compilation, atom, names_only=False):
self._node = compilation.hierarchy.find(atom)
if self._node is None:
raise ValueError("Unable to find atom '%s' in compilation"
" hierarchy" % atom)
raise ValueError(
"Unable to find atom '%s' in compilation hierarchy" % atom
)
self._level_cache = {}
self._atom = atom
self._execution_graph = compilation.execution_graph
@@ -78,8 +79,10 @@ class ScopeWalker:
graph = self._execution_graph
if self._predecessors is None:
predecessors = {
node for node in graph.bfs_predecessors_iter(self._atom)
if graph.nodes[node]['kind'] in co.ATOMS}
node
for node in graph.bfs_predecessors_iter(self._atom)
if graph.nodes[node]['kind'] in co.ATOMS
}
self._predecessors = predecessors.copy()
else:
predecessors = self._predecessors.copy()
@@ -95,7 +98,8 @@ class ScopeWalker:
visible = []
removals = set()
atom_it = tr.depth_first_reverse_iterate(
parent, start_from_idx=last_idx)
parent, start_from_idx=last_idx
)
for atom in atom_it:
if atom in predecessors:
predecessors.remove(atom)
@@ -106,9 +110,14 @@ class ScopeWalker:
self._level_cache[lvl] = (visible, removals)
if LOG.isEnabledFor(logging.TRACE):
visible_names = [a.name for a in visible]
LOG.trace("Scope visible to '%s' (limited by parent '%s'"
" index < %s) is: %s", self._atom,
parent.item.name, last_idx, visible_names)
LOG.trace(
"Scope visible to '%s' (limited by parent '%s'"
" index < %s) is: %s",
self._atom,
parent.item.name,
last_idx,
visible_names,
)
if self._names_only:
yield [a.name for a in visible]
else:

160
taskflow/engines/action_engine/selector.py
View File

@@ -43,16 +43,22 @@ class Selector:
def iter_next_atoms(self, atom=None):
"""Iterate next atoms to run (originating from atom or all atoms)."""
if atom is None:
return iter_utils.unique_seen((self._browse_atoms_for_execute(),
self._browse_atoms_for_revert()),
seen_selector=operator.itemgetter(0))
return iter_utils.unique_seen(
(
self._browse_atoms_for_execute(),
self._browse_atoms_for_revert(),
),
seen_selector=operator.itemgetter(0),
)
state = self._storage.get_atom_state(atom.name)
intention = self._storage.get_atom_intention(atom.name)
if state == st.SUCCESS:
if intention == st.REVERT:
return iter([
(atom, deciders.NoOpDecider()),
])
return iter(
[
(atom, deciders.NoOpDecider()),
]
)
elif intention == st.EXECUTE:
return self._browse_atoms_for_execute(atom=atom)
else:
@@ -82,7 +88,8 @@ class Selector:
# problematic to determine as top levels can have their deciders
# applied **after** going deeper).
atom_it = traversal.breadth_first_iterate(
self._execution_graph, atom, traversal.Direction.FORWARD)
self._execution_graph, atom, traversal.Direction.FORWARD
)
for atom in atom_it:
is_ready, late_decider = self._get_maybe_ready_for_execute(atom)
if is_ready:
@@ -100,22 +107,32 @@ class Selector:
atom_it = self._runtime.iterate_nodes(co.ATOMS)
else:
atom_it = traversal.breadth_first_iterate(
self._execution_graph, atom, traversal.Direction.BACKWARD,
self._execution_graph,
atom,
traversal.Direction.BACKWARD,
# Stop at the retry boundary (as retries 'control' there
# surronding atoms, and we don't want to back track over
# them so that they can correctly affect there associated
# atoms); we do though need to jump through all tasks since
# if a predecessor Y was ignored and a predecessor Z before Y
# was not it should be eligible to now revert...
through_retries=False)
through_retries=False,
)
for atom in atom_it:
is_ready, late_decider = self._get_maybe_ready_for_revert(atom)
if is_ready:
yield (atom, late_decider)
def _get_maybe_ready(self, atom, transition_to, allowed_intentions,
connected_fetcher, ready_checker,
decider_fetcher, for_what="?"):
def _get_maybe_ready(
self,
atom,
transition_to,
allowed_intentions,
connected_fetcher,
ready_checker,
decider_fetcher,
for_what="?",
):
def iter_connected_states():
# Lazily iterate over connected states so that ready checkers
# can stop early (vs having to consume and check all the
@@ -126,6 +143,7 @@ class Selector:
# to avoid two calls into storage).
atom_states = self._storage.get_atoms_states([atom.name])
yield (atom, atom_states[atom.name])
# NOTE(harlowja): How this works is the following...
#
# 1. First check if the current atom can even transition to the
@@ -144,18 +162,29 @@ class Selector:
# which can (if it desires) affect this ready result (but does
# so right before the atom is about to be scheduled).
state = self._storage.get_atom_state(atom.name)
ok_to_transition = self._runtime.check_atom_transition(atom, state,
transition_to)
ok_to_transition = self._runtime.check_atom_transition(
atom, state, transition_to
)
if not ok_to_transition:
LOG.trace("Atom '%s' is not ready to %s since it can not"
" transition to %s from its current state %s",
atom, for_what, transition_to, state)
LOG.trace(
"Atom '%s' is not ready to %s since it can not"
" transition to %s from its current state %s",
atom,
for_what,
transition_to,
state,
)
return (False, None)
intention = self._storage.get_atom_intention(atom.name)
if intention not in allowed_intentions:
LOG.trace("Atom '%s' is not ready to %s since its current"
" intention %s is not in allowed intentions %s",
atom, for_what, intention, allowed_intentions)
LOG.trace(
"Atom '%s' is not ready to %s since its current"
" intention %s is not in allowed intentions %s",
atom,
for_what,
intention,
allowed_intentions,
)
return (False, None)
ok_to_run = ready_checker(iter_connected_states())
if not ok_to_run:
@@ -165,62 +194,91 @@ class Selector:
def _get_maybe_ready_for_execute(self, atom):
"""Returns if an atom is *likely* ready to be executed."""
def ready_checker(pred_connected_it):
for pred in pred_connected_it:
pred_atom, (pred_atom_state, pred_atom_intention) = pred
if (pred_atom_state in (st.SUCCESS, st.IGNORE) and
pred_atom_intention in (st.EXECUTE, st.IGNORE)):
if pred_atom_state in (
st.SUCCESS,
st.IGNORE,
) and pred_atom_intention in (st.EXECUTE, st.IGNORE):
continue
LOG.trace("Unable to begin to execute since predecessor"
" atom '%s' is in state %s with intention %s",
pred_atom, pred_atom_state, pred_atom_intention)
LOG.trace(
"Unable to begin to execute since predecessor"
" atom '%s' is in state %s with intention %s",
pred_atom,
pred_atom_state,
pred_atom_intention,
)
return False
LOG.trace("Able to let '%s' execute", atom)
return True
decider_fetcher = lambda: \
deciders.IgnoreDecider(
atom, self._runtime.fetch_edge_deciders(atom))
connected_fetcher = lambda: \
traversal.depth_first_iterate(self._execution_graph, atom,
# Whether the desired atom
# can execute is dependent on its
# predecessors outcomes (thus why
# we look backwards).
traversal.Direction.BACKWARD)
decider_fetcher = lambda: deciders.IgnoreDecider(
atom, self._runtime.fetch_edge_deciders(atom)
)
connected_fetcher = lambda: traversal.depth_first_iterate(
self._execution_graph,
atom,
# Whether the desired atom
# can execute is dependent on its
# predecessors outcomes (thus why
# we look backwards).
traversal.Direction.BACKWARD,
)
# If this atoms current state is able to be transitioned to RUNNING
# and its intention is to EXECUTE and all of its predecessors executed
# successfully or were ignored then this atom is ready to execute.
LOG.trace("Checking if '%s' is ready to execute", atom)
return self._get_maybe_ready(atom, st.RUNNING, [st.EXECUTE],
connected_fetcher, ready_checker,
decider_fetcher, for_what='execute')
return self._get_maybe_ready(
atom,
st.RUNNING,
[st.EXECUTE],
connected_fetcher,
ready_checker,
decider_fetcher,
for_what='execute',
)
def _get_maybe_ready_for_revert(self, atom):
"""Returns if an atom is *likely* ready to be reverted."""
def ready_checker(succ_connected_it):
for succ in succ_connected_it:
succ_atom, (succ_atom_state, _succ_atom_intention) = succ
if succ_atom_state not in (st.PENDING, st.REVERTED, st.IGNORE):
LOG.trace("Unable to begin to revert since successor"
" atom '%s' is in state %s", succ_atom,
succ_atom_state)
LOG.trace(
"Unable to begin to revert since successor"
" atom '%s' is in state %s",
succ_atom,
succ_atom_state,
)
return False
LOG.trace("Able to let '%s' revert", atom)
return True
noop_decider = deciders.NoOpDecider()
connected_fetcher = lambda: \
traversal.depth_first_iterate(self._execution_graph, atom,
# Whether the desired atom
# can revert is dependent on its
# successors states (thus why we
# look forwards).
traversal.Direction.FORWARD)
connected_fetcher = lambda: traversal.depth_first_iterate(
self._execution_graph,
atom,
# Whether the desired atom
# can revert is dependent on its
# successors states (thus why we
# look forwards).
traversal.Direction.FORWARD,
)
decider_fetcher = lambda: noop_decider
# If this atoms current state is able to be transitioned to REVERTING
# and its intention is either REVERT or RETRY and all of its
# successors are either PENDING or REVERTED then this atom is ready
# to revert.
LOG.trace("Checking if '%s' is ready to revert", atom)
return self._get_maybe_ready(atom, st.REVERTING, [st.REVERT, st.RETRY],
connected_fetcher, ready_checker,
decider_fetcher, for_what='revert')
return self._get_maybe_ready(
atom,
st.REVERTING,
[st.REVERT, st.RETRY],
connected_fetcher,
ready_checker,
decider_fetcher,
for_what='revert',
)

53
taskflow/engines/action_engine/traversal.py
View File

@@ -28,9 +28,14 @@ class Direction(enum.Enum):
BACKWARD = 2
def _extract_connectors(execution_graph, starting_node, direction,
through_flows=True, through_retries=True,
through_tasks=True):
def _extract_connectors(
execution_graph,
starting_node,
direction,
through_flows=True,
through_retries=True,
through_tasks=True,
):
if direction == Direction.FORWARD:
connected_iter = execution_graph.successors
else:
@@ -46,9 +51,14 @@ def _extract_connectors(execution_graph, starting_node, direction,
return connected_iter(starting_node), connected_to_functors
def breadth_first_iterate(execution_graph, starting_node, direction,
through_flows=True, through_retries=True,
through_tasks=True):
def breadth_first_iterate(
execution_graph,
starting_node,
direction,
through_flows=True,
through_retries=True,
through_tasks=True,
):
"""Iterates connected nodes in execution graph (from starting node).
Does so in a breadth first manner.
@@ -56,9 +66,13 @@ def breadth_first_iterate(execution_graph, starting_node, direction,
Jumps over nodes with ``noop`` attribute (does not yield them back).
"""
initial_nodes_iter, connected_to_functors = _extract_connectors(
execution_graph, starting_node, direction,
through_flows=through_flows, through_retries=through_retries,
through_tasks=through_tasks)
execution_graph,
starting_node,
direction,
through_flows=through_flows,
through_retries=through_retries,
through_tasks=through_tasks,
)
q = collections.deque(initial_nodes_iter)
visited_nodes = set()
while q:
@@ -79,9 +93,14 @@ def breadth_first_iterate(execution_graph, starting_node, direction,
q.extend(connected_to_functor(node))
def depth_first_iterate(execution_graph, starting_node, direction,
through_flows=True, through_retries=True,
through_tasks=True):
def depth_first_iterate(
execution_graph,
starting_node,
direction,
through_flows=True,
through_retries=True,
through_tasks=True,
):
"""Iterates connected nodes in execution graph (from starting node).
Does so in a depth first manner.
@@ -89,9 +108,13 @@ def depth_first_iterate(execution_graph, starting_node, direction,
Jumps over nodes with ``noop`` attribute (does not yield them back).
"""
initial_nodes_iter, connected_to_functors = _extract_connectors(
execution_graph, starting_node, direction,
through_flows=through_flows, through_retries=through_retries,
through_tasks=through_tasks)
execution_graph,
starting_node,
direction,
through_flows=through_flows,
through_retries=through_retries,
through_tasks=through_tasks,
)
stack = list(initial_nodes_iter)
visited_nodes = set()
while stack:

144
taskflow/engines/helpers.py
View File

@@ -60,8 +60,9 @@ def _fetch_factory(factory_name):
try:
return importutils.import_class(factory_name)
except (ImportError, ValueError) as e:
raise ImportError("Could not import factory %r: %s"
% (factory_name, e))
raise ImportError(
"Could not import factory %r: %s" % (factory_name, e)
)
def _fetch_validate_factory(flow_factory):
@@ -73,16 +74,25 @@ def _fetch_validate_factory(flow_factory):
factory_name = reflection.get_callable_name(flow_factory)
try:
reimported = _fetch_factory(factory_name)
assert reimported == factory_fun
assert reimported == factory_fun # noqa: S101
except (ImportError, AssertionError):
raise ValueError('Flow factory %r is not reimportable by name %s'
% (factory_fun, factory_name))
raise ValueError(
'Flow factory %r is not reimportable by name %s'
% (factory_fun, factory_name)
)
return (factory_name, factory_fun)
def load(flow, store=None, flow_detail=None, book=None,
backend=None, namespace=ENGINES_NAMESPACE,
engine=ENGINE_DEFAULT, **kwargs):
def load(
flow,
store=None,
flow_detail=None,
book=None,
backend=None,
namespace=ENGINES_NAMESPACE,
engine=ENGINE_DEFAULT,
**kwargs,
):
"""Load a flow into an engine.
This function creates and prepares an engine to run the provided flow. All
@@ -122,15 +132,18 @@ def load(flow, store=None, flow_detail=None, book=None,
backend = p_backends.fetch(backend)
if flow_detail is None:
flow_detail = p_utils.create_flow_detail(flow, book=book,
backend=backend)
flow_detail = p_utils.create_flow_detail(
flow, book=book, backend=backend
)
LOG.debug('Looking for %r engine driver in %r', kind, namespace)
try:
mgr = stevedore.driver.DriverManager(
namespace, kind,
namespace,
kind,
invoke_on_load=True,
invoke_args=(flow, flow_detail, backend, options))
invoke_args=(flow, flow_detail, backend, options),
)
engine = mgr.driver
except RuntimeError as e:
raise exc.NotFound("Could not find engine '%s'" % (kind), e)
@@ -140,9 +153,16 @@ def load(flow, store=None, flow_detail=None, book=None,
return engine
def run(flow, store=None, flow_detail=None, book=None,
backend=None, namespace=ENGINES_NAMESPACE,
engine=ENGINE_DEFAULT, **kwargs):
def run(
flow,
store=None,
flow_detail=None,
book=None,
backend=None,
namespace=ENGINES_NAMESPACE,
engine=ENGINE_DEFAULT,
**kwargs,
):
"""Run the flow.
This function loads the flow into an engine (with the :func:`load() <load>`
@@ -153,16 +173,23 @@ def run(flow, store=None, flow_detail=None, book=None,
:returns: dictionary of all named
results (see :py:meth:`~.taskflow.storage.Storage.fetch_all`)
"""
engine = load(flow, store=store, flow_detail=flow_detail, book=book,
backend=backend, namespace=namespace,
engine=engine, **kwargs)
engine = load(
flow,
store=store,
flow_detail=flow_detail,
book=book,
backend=backend,
namespace=namespace,
engine=engine,
**kwargs,
)
engine.run()
return engine.storage.fetch_all()
def save_factory_details(flow_detail,
flow_factory, factory_args, factory_kwargs,
backend=None):
def save_factory_details(
flow_detail, flow_factory, factory_args, factory_kwargs, backend=None
):
"""Saves the given factories reimportable attributes into the flow detail.
This function saves the factory name, arguments, and keyword arguments
@@ -198,10 +225,17 @@ def save_factory_details(flow_detail,
conn.update_flow_details(flow_detail)
def load_from_factory(flow_factory, factory_args=None, factory_kwargs=None,
store=None, book=None, backend=None,
namespace=ENGINES_NAMESPACE, engine=ENGINE_DEFAULT,
**kwargs):
def load_from_factory(
flow_factory,
factory_args=None,
factory_kwargs=None,
store=None,
book=None,
backend=None,
namespace=ENGINES_NAMESPACE,
engine=ENGINE_DEFAULT,
**kwargs,
):
"""Loads a flow from a factory function into an engine.
Gets flow factory function (or name of it) and creates flow with
@@ -227,12 +261,23 @@ def load_from_factory(flow_factory, factory_args=None, factory_kwargs=None,
if isinstance(backend, dict):
backend = p_backends.fetch(backend)
flow_detail = p_utils.create_flow_detail(flow, book=book, backend=backend)
save_factory_details(flow_detail,
flow_factory, factory_args, factory_kwargs,
backend=backend)
return load(flow=flow, store=store, flow_detail=flow_detail, book=book,
backend=backend, namespace=namespace,
engine=engine, **kwargs)
save_factory_details(
flow_detail,
flow_factory,
factory_args,
factory_kwargs,
backend=backend,
)
return load(
flow=flow,
store=store,
flow_detail=flow_detail,
book=book,
backend=backend,
namespace=namespace,
engine=engine,
**kwargs,
)
def flow_from_detail(flow_detail):
@@ -247,24 +292,33 @@ def flow_from_detail(flow_detail):
try:
factory_data = flow_detail.meta['factory']
except (KeyError, AttributeError, TypeError):
raise ValueError('Cannot reconstruct flow %s %s: '
'no factory information saved.'
% (flow_detail.name, flow_detail.uuid))
raise ValueError(
'Cannot reconstruct flow %s %s: '
'no factory information saved.'
% (flow_detail.name, flow_detail.uuid)
)
try:
factory_fun = _fetch_factory(factory_data['name'])
except (KeyError, ImportError):
raise ImportError('Could not import factory for flow %s %s'
% (flow_detail.name, flow_detail.uuid))
raise ImportError(
'Could not import factory for flow %s %s'
% (flow_detail.name, flow_detail.uuid)
)
args = factory_data.get('args', ())
kwargs = factory_data.get('kwargs', {})
return factory_fun(*args, **kwargs)
def load_from_detail(flow_detail, store=None, backend=None,
namespace=ENGINES_NAMESPACE, engine=ENGINE_DEFAULT,
**kwargs):
def load_from_detail(
flow_detail,
store=None,
backend=None,
namespace=ENGINES_NAMESPACE,
engine=ENGINE_DEFAULT,
**kwargs,
):
"""Reloads an engine previously saved.
This reloads the flow using the
@@ -278,6 +332,12 @@ def load_from_detail(flow_detail, store=None, backend=None,
:returns: engine
"""
flow = flow_from_detail(flow_detail)
return load(flow, flow_detail=flow_detail,
store=store, backend=backend,
namespace=namespace, engine=engine, **kwargs)
return load(
flow,
flow_detail=flow_detail,
store=store,
backend=backend,
namespace=namespace,
engine=engine,
**kwargs,
)

72
taskflow/engines/worker_based/dispatcher.py
View File

@@ -100,9 +100,13 @@ class TypeDispatcher:
if cb(data, message):
requeue_votes += 1
except Exception:
LOG.exception("Failed calling requeue filter %s '%s' to"
" determine if message %r should be requeued.",
i + 1, cb, message.delivery_tag)
LOG.exception(
"Failed calling requeue filter %s '%s' to"
" determine if message %r should be requeued.",
i + 1,
cb,
message.delivery_tag,
)
return requeue_votes
def _requeue_log_error(self, message, errors):
@@ -114,52 +118,72 @@ class TypeDispatcher:
# This was taken from how kombu is formatting its messages
# when its reject_log_error or ack_log_error functions are
# used so that we have a similar error format for requeuing.
LOG.critical("Couldn't requeue %r, reason:%r",
message.delivery_tag, exc, exc_info=True)
LOG.critical(
"Couldn't requeue %r, reason:%r",
message.delivery_tag,
exc,
exc_info=True,
)
else:
LOG.debug("Message '%s' was requeued.", ku.DelayedPretty(message))
def _process_message(self, data, message, message_type):
handler = self._type_handlers.get(message_type)
if handler is None:
message.reject_log_error(logger=LOG,
errors=(kombu_exc.MessageStateError,))
LOG.warning("Unexpected message type: '%s' in message"
" '%s'", message_type, ku.DelayedPretty(message))
message.reject_log_error(
logger=LOG, errors=(kombu_exc.MessageStateError,)
)
LOG.warning(
"Unexpected message type: '%s' in message '%s'",
message_type,
ku.DelayedPretty(message),
)
else:
if handler.validator is not None:
try:
handler.validator(data)
except excp.InvalidFormat as e:
message.reject_log_error(
logger=LOG, errors=(kombu_exc.MessageStateError,))
LOG.warning("Message '%s' (%s) was rejected due to it"
" being in an invalid format: %s",
ku.DelayedPretty(message), message_type, e)
logger=LOG, errors=(kombu_exc.MessageStateError,)
)
LOG.warning(
"Message '%s' (%s) was rejected due to it"
" being in an invalid format: %s",
ku.DelayedPretty(message),
message_type,
e,
)
return
message.ack_log_error(logger=LOG,
errors=(kombu_exc.MessageStateError,))
message.ack_log_error(
logger=LOG, errors=(kombu_exc.MessageStateError,)
)
if message.acknowledged:
LOG.debug("Message '%s' was acknowledged.",
ku.DelayedPretty(message))
LOG.debug(
"Message '%s' was acknowledged.", ku.DelayedPretty(message)
)
handler.process_message(data, message)
else:
message.reject_log_error(logger=LOG,
errors=(kombu_exc.MessageStateError,))
message.reject_log_error(
logger=LOG, errors=(kombu_exc.MessageStateError,)
)
def on_message(self, data, message):
"""This method is called on incoming messages."""
LOG.debug("Received message '%s'", ku.DelayedPretty(message))
if self._collect_requeue_votes(data, message):
self._requeue_log_error(message,
errors=(kombu_exc.MessageStateError,))
self._requeue_log_error(
message, errors=(kombu_exc.MessageStateError,)
)
else:
try:
message_type = message.properties['type']
except KeyError:
message.reject_log_error(
logger=LOG, errors=(kombu_exc.MessageStateError,))
LOG.warning("The 'type' message property is missing"
" in message '%s'", ku.DelayedPretty(message))
logger=LOG, errors=(kombu_exc.MessageStateError,)
)
LOG.warning(
"The 'type' message property is missing in message '%s'",
ku.DelayedPretty(message),
)
else:
self._process_message(data, message, message_type)

21
taskflow/engines/worker_based/engine.py
View File

@@ -54,17 +54,20 @@ class WorkerBasedActionEngine(engine.ActionEngine):
super().__init__(flow, flow_detail, backend, options)
# This ensures that any provided executor will be validated before
# we get to far in the compilation/execution pipeline...
self._task_executor = self._fetch_task_executor(self._options,
self._flow_detail)
self._task_executor = self._fetch_task_executor(
self._options, self._flow_detail
)
@classmethod
def _fetch_task_executor(cls, options, flow_detail):
try:
e = options['executor']
if not isinstance(e, executor.WorkerTaskExecutor):
raise TypeError("Expected an instance of type '%s' instead of"
" type '%s' for 'executor' option"
% (executor.WorkerTaskExecutor, type(e)))
raise TypeError(
"Expected an instance of type '%s' instead of"
" type '%s' for 'executor' option"
% (executor.WorkerTaskExecutor, type(e))
)
return e
except KeyError:
return executor.WorkerTaskExecutor(
@@ -75,8 +78,8 @@ class WorkerBasedActionEngine(engine.ActionEngine):
topics=options.get('topics', []),
transport=options.get('transport'),
transport_options=options.get('transport_options'),
transition_timeout=options.get('transition_timeout',
pr.REQUEST_TIMEOUT),
worker_expiry=options.get('worker_expiry',
pr.EXPIRES_AFTER),
transition_timeout=options.get(
'transition_timeout', pr.REQUEST_TIMEOUT
),
worker_expiry=options.get('worker_expiry', pr.EXPIRES_AFTER),
)

215
taskflow/engines/worker_based/executor.py
View File

@@ -35,35 +35,53 @@ LOG = logging.getLogger(__name__)
class WorkerTaskExecutor(executor.TaskExecutor):
"""Executes tasks on remote workers."""
def __init__(self, uuid, exchange, topics,
transition_timeout=pr.REQUEST_TIMEOUT,
url=None, transport=None, transport_options=None,
retry_options=None, worker_expiry=pr.EXPIRES_AFTER):
def __init__(
self,
uuid,
exchange,
topics,
transition_timeout=pr.REQUEST_TIMEOUT,
url=None,
transport=None,
transport_options=None,
retry_options=None,
worker_expiry=pr.EXPIRES_AFTER,
):
self._uuid = uuid
self._ongoing_requests = {}
self._ongoing_requests_lock = threading.RLock()
self._transition_timeout = transition_timeout
self._proxy = proxy.Proxy(uuid, exchange,
on_wait=self._on_wait, url=url,
transport=transport,
transport_options=transport_options,
retry_options=retry_options)
self._proxy = proxy.Proxy(
uuid,
exchange,
on_wait=self._on_wait,
url=url,
transport=transport,
transport_options=transport_options,
retry_options=retry_options,
)
# NOTE(harlowja): This is the most simplest finder impl. that
# doesn't have external dependencies (outside of what this engine
# already requires); it though does create periodic 'polling' traffic
# to workers to 'learn' of the tasks they can perform (and requires
# pre-existing knowledge of the topics those workers are on to gather
# and update this information).
self._finder = wt.ProxyWorkerFinder(uuid, self._proxy, topics,
worker_expiry=worker_expiry)
self._proxy.dispatcher.type_handlers.update({
pr.RESPONSE: dispatcher.Handler(self._process_response,
validator=pr.Response.validate),
pr.NOTIFY: dispatcher.Handler(
self._finder.process_response,
validator=functools.partial(pr.Notify.validate,
response=True)),
})
self._finder = wt.ProxyWorkerFinder(
uuid, self._proxy, topics, worker_expiry=worker_expiry
)
self._proxy.dispatcher.type_handlers.update(
{
pr.RESPONSE: dispatcher.Handler(
self._process_response, validator=pr.Response.validate
),
pr.NOTIFY: dispatcher.Handler(
self._finder.process_response,
validator=functools.partial(
pr.Notify.validate, response=True
),
),
}
)
# Thread that will run the message dispatching (and periodically
# call the on_wait callback to do various things) loop...
self._helper = None
@@ -73,20 +91,27 @@ class WorkerTaskExecutor(executor.TaskExecutor):
def _process_response(self, response, message):
"""Process response from remote side."""
LOG.debug("Started processing response message '%s'",
ku.DelayedPretty(message))
LOG.debug(
"Started processing response message '%s'",
ku.DelayedPretty(message),
)
try:
request_uuid = message.properties['correlation_id']
except KeyError:
LOG.warning("The 'correlation_id' message property is"
" missing in message '%s'",
ku.DelayedPretty(message))
LOG.warning(
"The 'correlation_id' message property is"
" missing in message '%s'",
ku.DelayedPretty(message),
)
else:
request = self._ongoing_requests.get(request_uuid)
if request is not None:
response = pr.Response.from_dict(response)
LOG.debug("Extracted response '%s' and matched it to"
" request '%s'", response, request)
LOG.debug(
"Extracted response '%s' and matched it to request '%s'",
response,
request,
)
if response.state == pr.RUNNING:
request.transition_and_log_error(pr.RUNNING, logger=LOG)
elif response.state == pr.EVENT:
@@ -98,14 +123,16 @@ class WorkerTaskExecutor(executor.TaskExecutor):
details = response.data['details']
request.task.notifier.notify(event_type, details)
elif response.state in (pr.FAILURE, pr.SUCCESS):
if request.transition_and_log_error(response.state,
logger=LOG):
if request.transition_and_log_error(
response.state, logger=LOG
):
with self._ongoing_requests_lock:
del self._ongoing_requests[request.uuid]
request.set_result(result=response.data['result'])
else:
LOG.warning("Unexpected response status '%s'",
response.state)
LOG.warning(
"Unexpected response status '%s'", response.state
)
else:
LOG.debug("Request with id='%s' not found", request_uuid)
@@ -126,7 +153,8 @@ class WorkerTaskExecutor(executor.TaskExecutor):
raise exc.RequestTimeout(
"Request '%s' has expired after waiting for %0.2f"
" seconds for it to transition out of (%s) states"
% (request, request_age, ", ".join(pr.WAITING_STATES)))
% (request, request_age, ", ".join(pr.WAITING_STATES))
)
except exc.RequestTimeout:
with misc.capture_failure() as failure:
LOG.debug(failure.exception_str)
@@ -169,9 +197,9 @@ class WorkerTaskExecutor(executor.TaskExecutor):
while waiting_requests:
_request_uuid, request = waiting_requests.popitem()
worker = finder.get_worker_for_task(request.task)
if (worker is not None and
request.transition_and_log_error(pr.PENDING,
logger=LOG)):
if worker is not None and request.transition_and_log_error(
pr.PENDING, logger=LOG
):
self._publish_request(request, worker)
self._messages_processed['finder'] = new_messages_processed
@@ -187,20 +215,36 @@ class WorkerTaskExecutor(executor.TaskExecutor):
# a worker located).
self._clean()
def _submit_task(self, task, task_uuid, action, arguments,
progress_callback=None, result=pr.NO_RESULT,
failures=None):
def _submit_task(
self,
task,
task_uuid,
action,
arguments,
progress_callback=None,
result=pr.NO_RESULT,
failures=None,
):
"""Submit task request to a worker."""
request = pr.Request(task, task_uuid, action, arguments,
timeout=self._transition_timeout,
result=result, failures=failures)
request = pr.Request(
task,
task_uuid,
action,
arguments,
timeout=self._transition_timeout,
result=result,
failures=failures,
)
# Register the callback, so that we can proxy the progress correctly.
if (progress_callback is not None and
task.notifier.can_be_registered(EVENT_UPDATE_PROGRESS)):
if progress_callback is not None and task.notifier.can_be_registered(
EVENT_UPDATE_PROGRESS
):
task.notifier.register(EVENT_UPDATE_PROGRESS, progress_callback)
request.future.add_done_callback(
lambda _fut: task.notifier.deregister(EVENT_UPDATE_PROGRESS,
progress_callback))
lambda _fut: task.notifier.deregister(
EVENT_UPDATE_PROGRESS, progress_callback
)
)
# Get task's worker and publish request if worker was found.
worker = self._finder.get_worker_for_task(task)
if worker is not None:
@@ -209,42 +253,75 @@ class WorkerTaskExecutor(executor.TaskExecutor):
self._ongoing_requests[request.uuid] = request
self._publish_request(request, worker)
else:
LOG.debug("Delaying submission of '%s', no currently known"
" worker/s available to process it", request)
LOG.debug(
"Delaying submission of '%s', no currently known"
" worker/s available to process it",
request,
)
with self._ongoing_requests_lock:
self._ongoing_requests[request.uuid] = request
return request.future
def _publish_request(self, request, worker):
"""Publish request to a given topic."""
LOG.debug("Submitting execution of '%s' to worker '%s' (expecting"
" response identified by reply_to=%s and"
" correlation_id=%s) - waited %0.3f seconds to"
" get published", request, worker, self._uuid,
request.uuid, timeutils.now() - request.created_on)
LOG.debug(
"Submitting execution of '%s' to worker '%s' (expecting"
" response identified by reply_to=%s and"
" correlation_id=%s) - waited %0.3f seconds to"
" get published",
request,
worker,
self._uuid,
request.uuid,
timeutils.now() - request.created_on,
)
try:
self._proxy.publish(request, worker.topic,
reply_to=self._uuid,
correlation_id=request.uuid)
self._proxy.publish(
request,
worker.topic,
reply_to=self._uuid,
correlation_id=request.uuid,
)
except Exception:
with misc.capture_failure() as failure:
LOG.critical("Failed to submit '%s' (transitioning it to"
" %s)", request, pr.FAILURE, exc_info=True)
LOG.critical(
"Failed to submit '%s' (transitioning it to %s)",
request,
pr.FAILURE,
exc_info=True,
)
if request.transition_and_log_error(pr.FAILURE, logger=LOG):
with self._ongoing_requests_lock:
del self._ongoing_requests[request.uuid]
request.set_result(failure)
def execute_task(self, task, task_uuid, arguments,
progress_callback=None):
return self._submit_task(task, task_uuid, pr.EXECUTE, arguments,
progress_callback=progress_callback)
def execute_task(self, task, task_uuid, arguments, progress_callback=None):
return self._submit_task(
task,
task_uuid,
pr.EXECUTE,
arguments,
progress_callback=progress_callback,
)
def revert_task(self, task, task_uuid, arguments, result, failures,
progress_callback=None):
return self._submit_task(task, task_uuid, pr.REVERT, arguments,
result=result, failures=failures,
progress_callback=progress_callback)
def revert_task(
self,
task,
task_uuid,
arguments,
result,
failures,
progress_callback=None,
):
return self._submit_task(
task,
task_uuid,
pr.REVERT,
arguments,
result=result,
failures=failures,
progress_callback=progress_callback,
)
def wait_for_workers(self, workers=1, timeout=None):
"""Waits for geq workers to notify they are ready to do work.
@@ -255,14 +332,14 @@ class WorkerTaskExecutor(executor.TaskExecutor):
return how many workers are still needed, otherwise it will
return zero.
"""
return self._finder.wait_for_workers(workers=workers,
timeout=timeout)
return self._finder.wait_for_workers(workers=workers, timeout=timeout)
def start(self):
"""Starts message processing thread."""
if self._helper is not None:
raise RuntimeError("Worker executor must be stopped before"
" it can be started")
raise RuntimeError(
"Worker executor must be stopped before it can be started"
)
self._helper = tu.daemon_thread(self._proxy.start)
self._helper.start()
self._proxy.wait()

100
taskflow/engines/worker_based/protocol.py
View File

@@ -53,10 +53,7 @@ EXECUTE = 'execute'
REVERT = 'revert'
# Remote task action to event map.
ACTION_TO_EVENT = {
EXECUTE: executor.EXECUTED,
REVERT: executor.REVERTED
}
ACTION_TO_EVENT = {EXECUTE: executor.EXECUTED, REVERT: executor.REVERTED}
# NOTE(skudriashev): A timeout which specifies request expiration period.
REQUEST_TIMEOUT = 60
@@ -149,9 +146,11 @@ class Message(metaclass=abc.ABCMeta):
"""Base class for all message types."""
def __repr__(self):
return ("<%s object at 0x%x with contents %s>"
% (reflection.get_class_name(self, fully_qualified=False),
id(self), self.to_dict()))
return "<%s object at 0x%x with contents %s>" % (
reflection.get_class_name(self, fully_qualified=False),
id(self),
self.to_dict(),
)
@abc.abstractmethod
def to_dict(self):
@@ -180,7 +179,7 @@ class Notify(Message):
"items": {
"type": "string",
},
}
},
},
"required": ["topic", 'tasks'],
"additionalProperties": False,
@@ -217,21 +216,24 @@ class Notify(Message):
except su.ValidationError as e:
cls_name = reflection.get_class_name(cls, fully_qualified=False)
if response:
excp.raise_with_cause(excp.InvalidFormat,
"%s message response data not of the"
" expected format: %s" % (cls_name,
e.message),
cause=e)
excp.raise_with_cause(
excp.InvalidFormat,
"%s message response data not of the"
" expected format: %s" % (cls_name, e.message),
cause=e,
)
else:
excp.raise_with_cause(excp.InvalidFormat,
"%s message sender data not of the"
" expected format: %s" % (cls_name,
e.message),
cause=e)
excp.raise_with_cause(
excp.InvalidFormat,
"%s message sender data not of the"
" expected format: %s" % (cls_name, e.message),
cause=e,
)
_WorkUnit = collections.namedtuple('_WorkUnit', ['task_cls', 'task_name',
'action', 'arguments'])
_WorkUnit = collections.namedtuple(
'_WorkUnit', ['task_cls', 'task_name', 'action', 'arguments']
)
class Request(Message):
@@ -299,9 +301,16 @@ class Request(Message):
'required': ['task_cls', 'task_name', 'task_version', 'action'],
}
def __init__(self, task, uuid, action,
arguments, timeout=REQUEST_TIMEOUT, result=NO_RESULT,
failures=None):
def __init__(
self,
task,
uuid,
action,
arguments,
timeout=REQUEST_TIMEOUT,
result=NO_RESULT,
failures=None,
):
self._action = action
self._event = ACTION_TO_EVENT[action]
self._arguments = arguments
@@ -383,8 +392,12 @@ class Request(Message):
try:
moved = self.transition(new_state)
except excp.InvalidState:
logger.warnng("Failed to transition '%s' to %s state.", self,
new_state, exc_info=True)
logger.warnng(
"Failed to transition '%s' to %s state.",
self,
new_state,
exc_info=True,
)
return moved
@fasteners.locked
@@ -402,14 +415,19 @@ class Request(Message):
try:
self._machine.process_event(make_an_event(new_state))
except (machine_excp.NotFound, machine_excp.InvalidState) as e:
raise excp.InvalidState("Request transition from %s to %s is"
" not allowed: %s" % (old_state,
new_state, e))
raise excp.InvalidState(
"Request transition from %s to %s is"
" not allowed: %s" % (old_state, new_state, e)
)
else:
if new_state in STOP_TIMER_STATES:
self._watch.stop()
LOG.debug("Transitioned '%s' from %s state to %s state", self,
old_state, new_state)
LOG.debug(
"Transitioned '%s' from %s state to %s state",
self,
old_state,
new_state,
)
return True
@classmethod
@@ -418,11 +436,12 @@ class Request(Message):
su.schema_validate(data, cls.SCHEMA)
except su.ValidationError as e:
cls_name = reflection.get_class_name(cls, fully_qualified=False)
excp.raise_with_cause(excp.InvalidFormat,
"%s message response data not of the"
" expected format: %s" % (cls_name,
e.message),
cause=e)
excp.raise_with_cause(
excp.InvalidFormat,
"%s message response data not of the"
" expected format: %s" % (cls_name, e.message),
cause=e,
)
else:
# Validate all failure dictionaries that *may* be present...
failures = []
@@ -556,11 +575,12 @@ class Response(Message):
su.schema_validate(data, cls.SCHEMA)
except su.ValidationError as e:
cls_name = reflection.get_class_name(cls, fully_qualified=False)
excp.raise_with_cause(excp.InvalidFormat,
"%s message response data not of the"
" expected format: %s" % (cls_name,
e.message),
cause=e)
excp.raise_with_cause(
excp.InvalidFormat,
"%s message response data not of the"
" expected format: %s" % (cls_name, e.message),
cause=e,
)
else:
state = data['state']
if state == FAILURE and 'result' in data:

102
taskflow/engines/worker_based/proxy.py
View File

@@ -31,11 +31,13 @@ DRAIN_EVENTS_PERIOD = 1
# instead of returning the raw results from the kombu connection objects
# themselves so that a person can not mutate those objects (which would be
# bad).
_ConnectionDetails = collections.namedtuple('_ConnectionDetails',
['uri', 'transport'])
_TransportDetails = collections.namedtuple('_TransportDetails',
['options', 'driver_type',
'driver_name', 'driver_version'])
_ConnectionDetails = collections.namedtuple(
'_ConnectionDetails', ['uri', 'transport']
)
_TransportDetails = collections.namedtuple(
'_TransportDetails',
['options', 'driver_type', 'driver_name', 'driver_version'],
)
class Proxy:
@@ -65,10 +67,17 @@ class Proxy:
# value is valid...
_RETRY_INT_OPTS = frozenset(['max_retries'])
def __init__(self, topic, exchange,
type_handlers=None, on_wait=None, url=None,
transport=None, transport_options=None,
retry_options=None):
def __init__(
self,
topic,
exchange,
type_handlers=None,
on_wait=None,
url=None,
transport=None,
transport_options=None,
retry_options=None,
):
self._topic = topic
self._exchange_name = exchange
self._on_wait = on_wait
@@ -77,7 +86,8 @@ class Proxy:
# NOTE(skudriashev): Process all incoming messages only if proxy is
# running, otherwise requeue them.
requeue_filters=[lambda data, message: not self.is_running],
type_handlers=type_handlers)
type_handlers=type_handlers,
)
ensure_options = self.DEFAULT_RETRY_OPTIONS.copy()
if retry_options is not None:
@@ -91,9 +101,11 @@ class Proxy:
else:
tmp_val = float(val)
if tmp_val < 0:
raise ValueError("Expected value greater or equal to"
" zero for 'retry_options' %s; got"
" %s instead" % (k, val))
raise ValueError(
"Expected value greater or equal to"
" zero for 'retry_options' %s; got"
" %s instead" % (k, val)
)
ensure_options[k] = tmp_val
self._ensure_options = ensure_options
@@ -104,12 +116,14 @@ class Proxy:
self._drain_events_timeout = polling_interval
# create connection
self._conn = kombu.Connection(url, transport=transport,
transport_options=transport_options)
self._conn = kombu.Connection(
url, transport=transport, transport_options=transport_options
)
# create exchange
self._exchange = kombu.Exchange(name=self._exchange_name,
durable=False, auto_delete=True)
self._exchange = kombu.Exchange(
name=self._exchange_name, durable=False, auto_delete=True
)
@property
def dispatcher(self):
@@ -131,10 +145,11 @@ class Proxy:
options=transport_options,
driver_type=self._conn.transport.driver_type,
driver_name=self._conn.transport.driver_name,
driver_version=driver_version)
driver_version=driver_version,
)
return _ConnectionDetails(
uri=self._conn.as_uri(include_password=False),
transport=transport)
uri=self._conn.as_uri(include_password=False), transport=transport
)
@property
def is_running(self):
@@ -144,10 +159,14 @@ class Proxy:
def _make_queue(self, routing_key, exchange, channel=None):
"""Make a named queue for the given exchange."""
queue_name = f"{self._exchange_name}_{routing_key}"
return kombu.Queue(name=queue_name,
routing_key=routing_key, durable=False,
exchange=exchange, auto_delete=True,
channel=channel)
return kombu.Queue(
name=queue_name,
routing_key=routing_key,
durable=False,
exchange=exchange,
auto_delete=True,
channel=channel,
)
def publish(self, msg, routing_key, reply_to=None, correlation_id=None):
"""Publish message to the named exchange with given routing key."""
@@ -159,27 +178,33 @@ class Proxy:
# Filter out any empty keys...
routing_keys = [r_k for r_k in routing_keys if r_k]
if not routing_keys:
LOG.warning("No routing key/s specified; unable to send '%s'"
" to any target queue on exchange '%s'", msg,
self._exchange_name)
LOG.warning(
"No routing key/s specified; unable to send '%s'"
" to any target queue on exchange '%s'",
msg,
self._exchange_name,
)
return
def _publish(producer, routing_key):
queue = self._make_queue(routing_key, self._exchange)
producer.publish(body=msg.to_dict(),
routing_key=routing_key,
exchange=self._exchange,
declare=[queue],
type=msg.TYPE,
reply_to=reply_to,
correlation_id=correlation_id)
producer.publish(
body=msg.to_dict(),
routing_key=routing_key,
exchange=self._exchange,
declare=[queue],
type=msg.TYPE,
reply_to=reply_to,
correlation_id=correlation_id,
)
def _publish_errback(exc, interval):
LOG.exception('Publishing error: %s', exc)
LOG.info('Retry triggering in %s seconds', interval)
LOG.debug("Sending '%s' message using routing keys %s",
msg, routing_keys)
LOG.debug(
"Sending '%s' message using routing keys %s", msg, routing_keys
)
with kombu.connections[self._conn].acquire(block=True) as conn:
with conn.Producer() as producer:
ensure_kwargs = self._ensure_options.copy()
@@ -201,8 +226,9 @@ class Proxy:
LOG.exception('Draining error: %s', exc)
LOG.info('Retry triggering in %s seconds', interval)
LOG.info("Starting to consume from the '%s' exchange.",
self._exchange_name)
LOG.info(
"Starting to consume from the '%s' exchange.", self._exchange_name
)
with kombu.connections[self._conn].acquire(block=True) as conn:
queue = self._make_queue(self._topic, self._exchange, channel=conn)
callbacks = [self._dispatcher.on_message]

198
taskflow/engines/worker_based/server.py
View File

@@ -32,27 +32,41 @@ LOG = logging.getLogger(__name__)
class Server:
"""Server implementation that waits for incoming tasks requests."""
def __init__(self, topic, exchange, executor, endpoints,
url=None, transport=None, transport_options=None,
retry_options=None):
def __init__(
self,
topic,
exchange,
executor,
endpoints,
url=None,
transport=None,
transport_options=None,
retry_options=None,
):
type_handlers = {
pr.NOTIFY: dispatcher.Handler(
self._delayed_process(self._process_notify),
validator=functools.partial(pr.Notify.validate,
response=False)),
validator=functools.partial(
pr.Notify.validate, response=False
),
),
pr.REQUEST: dispatcher.Handler(
self._delayed_process(self._process_request),
validator=pr.Request.validate),
validator=pr.Request.validate,
),
}
self._executor = executor
self._proxy = proxy.Proxy(topic, exchange,
type_handlers=type_handlers,
url=url, transport=transport,
transport_options=transport_options,
retry_options=retry_options)
self._proxy = proxy.Proxy(
topic,
exchange,
type_handlers=type_handlers,
url=url,
transport=transport,
transport_options=transport_options,
retry_options=retry_options,
)
self._topic = topic
self._endpoints = {endpoint.name: endpoint
for endpoint in endpoints}
self._endpoints = {endpoint.name: endpoint for endpoint in endpoints}
def _delayed_process(self, func):
"""Runs the function using the instances executor (eventually).
@@ -65,25 +79,34 @@ class Server:
func_name = reflection.get_callable_name(func)
def _on_run(watch, content, message):
LOG.trace("It took %s seconds to get around to running"
" function/method '%s' with"
" message '%s'", watch.elapsed(), func_name,
ku.DelayedPretty(message))
LOG.trace(
"It took %s seconds to get around to running"
" function/method '%s' with"
" message '%s'",
watch.elapsed(),
func_name,
ku.DelayedPretty(message),
)
return func(content, message)
def _on_receive(content, message):
LOG.debug("Submitting message '%s' for execution in the"
" future to '%s'", ku.DelayedPretty(message), func_name)
LOG.debug(
"Submitting message '%s' for execution in the future to '%s'",
ku.DelayedPretty(message),
func_name,
)
watch = timeutils.StopWatch()
watch.start()
try:
self._executor.submit(_on_run, watch, content, message)
except RuntimeError:
LOG.error("Unable to continue processing message '%s',"
" submission to instance executor (with later"
" execution by '%s') was unsuccessful",
ku.DelayedPretty(message), func_name,
exc_info=True)
LOG.exception(
"Unable to continue processing message '%s',"
" submission to instance executor (with later"
" execution by '%s') was unsuccessful",
ku.DelayedPretty(message),
func_name,
)
return _on_receive
@@ -103,8 +126,7 @@ class Server:
try:
properties.append(message.properties[prop])
except KeyError:
raise ValueError("The '%s' message property is missing" %
prop)
raise ValueError("The '%s' message property is missing" % prop)
return properties
def _reply(self, capture, reply_to, task_uuid, state=pr.FAILURE, **kwargs):
@@ -122,9 +144,13 @@ class Server:
except Exception:
if not capture:
raise
LOG.critical("Failed to send reply to '%s' for task '%s' with"
" response %s", reply_to, task_uuid, response,
exc_info=True)
LOG.critical(
"Failed to send reply to '%s' for task '%s' with response %s",
reply_to,
task_uuid,
response,
exc_info=True,
)
return published
def _on_event(self, reply_to, task_uuid, event_type, details):
@@ -132,26 +158,39 @@ class Server:
# NOTE(harlowja): the executor that will trigger this using the
# task notification/listener mechanism will handle logging if this
# fails, so thats why capture is 'False' is used here.
self._reply(False, reply_to, task_uuid, pr.EVENT,
event_type=event_type, details=details)
self._reply(
False,
reply_to,
task_uuid,
pr.EVENT,
event_type=event_type,
details=details,
)
def _process_notify(self, notify, message):
"""Process notify message and reply back."""
try:
reply_to = message.properties['reply_to']
except KeyError:
LOG.warning("The 'reply_to' message property is missing"
" in received notify message '%s'",
ku.DelayedPretty(message), exc_info=True)
LOG.warning(
"The 'reply_to' message property is missing"
" in received notify message '%s'",
ku.DelayedPretty(message),
exc_info=True,
)
else:
response = pr.Notify(topic=self._topic,
tasks=list(self._endpoints.keys()))
response = pr.Notify(
topic=self._topic, tasks=list(self._endpoints.keys())
)
try:
self._proxy.publish(response, routing_key=reply_to)
except Exception:
LOG.critical("Failed to send reply to '%s' with notify"
" response '%s'", reply_to, response,
exc_info=True)
LOG.critical(
"Failed to send reply to '%s' with notify response '%s'",
reply_to,
response,
exc_info=True,
)
def _process_request(self, request, message):
"""Process request message and reply back."""
@@ -162,22 +201,28 @@ class Server:
# in the first place...).
reply_to, task_uuid = self._parse_message(message)
except ValueError:
LOG.warning("Failed to parse request attributes from message '%s'",
ku.DelayedPretty(message), exc_info=True)
LOG.warning(
"Failed to parse request attributes from message '%s'",
ku.DelayedPretty(message),
exc_info=True,
)
return
else:
# prepare reply callback
reply_callback = functools.partial(self._reply, True, reply_to,
task_uuid)
reply_callback = functools.partial(
self._reply, True, reply_to, task_uuid
)
# Parse the request to get the activity/work to perform.
try:
work = pr.Request.from_dict(request, task_uuid=task_uuid)
except ValueError:
with misc.capture_failure() as failure:
LOG.warning("Failed to parse request contents"
" from message '%s'",
ku.DelayedPretty(message), exc_info=True)
LOG.warning(
"Failed to parse request contents from message '%s'",
ku.DelayedPretty(message),
exc_info=True,
)
reply_callback(result=pr.failure_to_dict(failure))
return
@@ -186,10 +231,13 @@ class Server:
endpoint = self._endpoints[work.task_cls]
except KeyError:
with misc.capture_failure() as failure:
LOG.warning("The '%s' task endpoint does not exist, unable"
" to continue processing request message '%s'",
work.task_cls, ku.DelayedPretty(message),
exc_info=True)
LOG.warning(
"The '%s' task endpoint does not exist, unable"
" to continue processing request message '%s'",
work.task_cls,
ku.DelayedPretty(message),
exc_info=True,
)
reply_callback(result=pr.failure_to_dict(failure))
return
else:
@@ -197,10 +245,15 @@ class Server:
handler = getattr(endpoint, work.action)
except AttributeError:
with misc.capture_failure() as failure:
LOG.warning("The '%s' handler does not exist on task"
" endpoint '%s', unable to continue processing"
" request message '%s'", work.action, endpoint,
ku.DelayedPretty(message), exc_info=True)
LOG.warning(
"The '%s' handler does not exist on task"
" endpoint '%s', unable to continue processing"
" request message '%s'",
work.action,
endpoint,
ku.DelayedPretty(message),
exc_info=True,
)
reply_callback(result=pr.failure_to_dict(failure))
return
else:
@@ -208,10 +261,14 @@ class Server:
task = endpoint.generate(name=work.task_name)
except Exception:
with misc.capture_failure() as failure:
LOG.warning("The '%s' task '%s' generation for request"
" message '%s' failed", endpoint,
work.action, ku.DelayedPretty(message),
exc_info=True)
LOG.warning(
"The '%s' task '%s' generation for request"
" message '%s' failed",
endpoint,
work.action,
ku.DelayedPretty(message),
exc_info=True,
)
reply_callback(result=pr.failure_to_dict(failure))
return
else:
@@ -222,24 +279,31 @@ class Server:
# emit them back to the engine... for handling at the engine side
# of things...
if task.notifier.can_be_registered(nt.Notifier.ANY):
task.notifier.register(nt.Notifier.ANY,
functools.partial(self._on_event,
reply_to, task_uuid))
task.notifier.register(
nt.Notifier.ANY,
functools.partial(self._on_event, reply_to, task_uuid),
)
elif isinstance(task.notifier, nt.RestrictedNotifier):
# Only proxy the allowable events then...
for event_type in task.notifier.events_iter():
task.notifier.register(event_type,
functools.partial(self._on_event,
reply_to, task_uuid))
task.notifier.register(
event_type,
functools.partial(self._on_event, reply_to, task_uuid),
)
# Perform the task action.
try:
result = handler(task, **work.arguments)
except Exception:
with misc.capture_failure() as failure:
LOG.warning("The '%s' endpoint '%s' execution for request"
" message '%s' failed", endpoint, work.action,
ku.DelayedPretty(message), exc_info=True)
LOG.warning(
"The '%s' endpoint '%s' execution for request"
" message '%s' failed",
endpoint,
work.action,
ku.DelayedPretty(message),
exc_info=True,
)
reply_callback(result=pr.failure_to_dict(failure))
else:
# And be done with it!

58
taskflow/engines/worker_based/types.py
View File

@@ -71,19 +71,26 @@ class TopicWorker:
r = reflection.get_class_name(self, fully_qualified=False)
if self.identity is not self._NO_IDENTITY:
r += "(identity={}, tasks={}, topic={})".format(
self.identity, self.tasks, self.topic)
self.identity, self.tasks, self.topic
)
else:
r += "(identity=*, tasks={}, topic={})".format(
self.tasks, self.topic)
self.tasks, self.topic
)
return r
class ProxyWorkerFinder:
"""Requests and receives responses about workers topic+task details."""
def __init__(self, uuid, proxy, topics,
beat_periodicity=pr.NOTIFY_PERIOD,
worker_expiry=pr.EXPIRES_AFTER):
def __init__(
self,
uuid,
proxy,
topics,
beat_periodicity=pr.NOTIFY_PERIOD,
worker_expiry=pr.EXPIRES_AFTER,
):
self._cond = threading.Condition()
self._proxy = proxy
self._topics = topics
@@ -134,7 +141,7 @@ class ProxyWorkerFinder:
if len(available_workers) == 1:
return available_workers[0]
else:
return random.choice(available_workers)
return random.choice(available_workers) # noqa: S311
@property
def messages_processed(self):
@@ -157,14 +164,14 @@ class ProxyWorkerFinder:
match workers to tasks to run).
"""
if self._messages_published == 0:
self._proxy.publish(pr.Notify(),
self._topics, reply_to=self._uuid)
self._proxy.publish(pr.Notify(), self._topics, reply_to=self._uuid)
self._messages_published += 1
self._watch.restart()
else:
if self._watch.expired():
self._proxy.publish(pr.Notify(),
self._topics, reply_to=self._uuid)
self._proxy.publish(
pr.Notify(), self._topics, reply_to=self._uuid
)
self._messages_published += 1
self._watch.restart()
@@ -188,16 +195,21 @@ class ProxyWorkerFinder:
def process_response(self, data, message):
"""Process notify message sent from remote side."""
LOG.debug("Started processing notify response message '%s'",
ku.DelayedPretty(message))
LOG.debug(
"Started processing notify response message '%s'",
ku.DelayedPretty(message),
)
response = pr.Notify(**data)
LOG.debug("Extracted notify response '%s'", response)
with self._cond:
worker, new_or_updated = self._add(response.topic,
response.tasks)
worker, new_or_updated = self._add(response.topic, response.tasks)
if new_or_updated:
LOG.debug("Updated worker '%s' (%s total workers are"
" currently known)", worker, self.total_workers)
LOG.debug(
"Updated worker '%s' (%s total workers are"
" currently known)",
worker,
self.total_workers,
)
self._cond.notify_all()
worker.last_seen = timeutils.now()
self._messages_processed += 1
@@ -207,8 +219,9 @@ class ProxyWorkerFinder:
Returns how many workers were removed.
"""
if (not self._workers or
(self._worker_expiry is None or self._worker_expiry <= 0)):
if not self._workers or (
self._worker_expiry is None or self._worker_expiry <= 0
):
return 0
dead_workers = {}
with self._cond:
@@ -225,9 +238,12 @@ class ProxyWorkerFinder:
self._cond.notify_all()
if dead_workers and LOG.isEnabledFor(logging.INFO):
for worker, secs_since_last_seen in dead_workers.values():
LOG.info("Removed worker '%s' as it has not responded to"
" notification requests in %0.3f seconds",
worker, secs_since_last_seen)
LOG.info(
"Removed worker '%s' as it has not responded to"
" notification requests in %0.3f seconds",
worker,
secs_since_last_seen,
)
return len(dead_workers)
def reset(self):

43
taskflow/engines/worker_based/worker.py
View File

@@ -55,24 +55,38 @@ class Worker:
(see: :py:attr:`~.proxy.Proxy.DEFAULT_RETRY_OPTIONS`)
"""
def __init__(self, exchange, topic, tasks,
executor=None, threads_count=None, url=None,
transport=None, transport_options=None,
retry_options=None):
def __init__(
self,
exchange,
topic,
tasks,
executor=None,
threads_count=None,
url=None,
transport=None,
transport_options=None,
retry_options=None,
):
self._topic = topic
self._executor = executor
self._owns_executor = False
if self._executor is None:
self._executor = futurist.ThreadPoolExecutor(
max_workers=threads_count)
max_workers=threads_count
)
self._owns_executor = True
self._endpoints = self._derive_endpoints(tasks)
self._exchange = exchange
self._server = server.Server(topic, exchange, self._executor,
self._endpoints, url=url,
transport=transport,
transport_options=transport_options,
retry_options=retry_options)
self._server = server.Server(
topic,
exchange,
self._executor,
self._endpoints,
url=url,
transport=transport,
transport_options=transport_options,
retry_options=retry_options,
)
@staticmethod
def _derive_endpoints(tasks):
@@ -86,8 +100,9 @@ class Worker:
connection_details = self._server.connection_details
transport = connection_details.transport
if transport.driver_version:
transport_driver = "{} v{}".format(transport.driver_name,
transport.driver_version)
transport_driver = "{} v{}".format(
transport.driver_name, transport.driver_version
)
else:
transport_driver = transport.driver_name
try:
@@ -145,12 +160,12 @@ class Worker:
if __name__ == '__main__':
import argparse
import logging as log
parser = argparse.ArgumentParser()
parser.add_argument("--exchange", required=True)
parser.add_argument("--connection-url", required=True)
parser.add_argument("--topic", required=True)
parser.add_argument("--task", action='append',
metavar="TASK", default=[])
parser.add_argument("--task", action='append', metavar="TASK", default=[])
parser.add_argument("-v", "--verbose", action='store_true')
args = parser.parse_args()
if args.verbose:

59
taskflow/examples/99_bottles.py
View File

@@ -22,9 +22,9 @@ import traceback
from kazoo import client
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow.conductors import backends as conductor_backends
@@ -90,16 +90,19 @@ def make_bottles(count):
s = lf.Flow("bottle-song")
take_bottle = TakeABottleDown("take-bottle-%s" % count,
inject={'bottles_left': count},
provides='bottles_left')
take_bottle = TakeABottleDown(
"take-bottle-%s" % count,
inject={'bottles_left': count},
provides='bottles_left',
)
pass_it = PassItAround("pass-%s-around" % count)
next_bottles = Conclusion("next-bottles-%s" % (count - 1))
s.add(take_bottle, pass_it, next_bottles)
for bottle in reversed(list(range(1, count))):
take_bottle = TakeABottleDown("take-bottle-%s" % bottle,
provides='bottles_left')
take_bottle = TakeABottleDown(
"take-bottle-%s" % bottle, provides='bottles_left'
)
pass_it = PassItAround("pass-%s-around" % bottle)
next_bottles = Conclusion("next-bottles-%s" % (bottle - 1))
s.add(take_bottle, pass_it, next_bottles)
@@ -122,15 +125,17 @@ def run_conductor(only_run_once=False):
if event.endswith("_start"):
w = timeutils.StopWatch()
w.start()
base_event = event[0:-len("_start")]
base_event = event[0 : -len("_start")]
event_watches[base_event] = w
if event.endswith("_end"):
base_event = event[0:-len("_end")]
base_event = event[0 : -len("_end")]
try:
w = event_watches.pop(base_event)
w.stop()
print("It took %0.3f seconds for event '%s' to finish"
% (w.elapsed(), base_event))
print(
"It took %0.3f seconds for event '%s' to finish"
% (w.elapsed(), base_event)
)
except KeyError:
pass
if event == 'running_end' and only_run_once:
@@ -142,12 +147,14 @@ def run_conductor(only_run_once=False):
with contextlib.closing(persist_backend):
with contextlib.closing(persist_backend.get_connection()) as conn:
conn.upgrade()
job_backend = job_backends.fetch(my_name, JB_CONF,
persistence=persist_backend)
job_backend = job_backends.fetch(
my_name, JB_CONF, persistence=persist_backend
)
job_backend.connect()
with contextlib.closing(job_backend):
cond = conductor_backends.fetch('blocking', my_name, job_backend,
persistence=persist_backend)
cond = conductor_backends.fetch(
'blocking', my_name, job_backend, persistence=persist_backend
)
on_conductor_event = functools.partial(on_conductor_event, cond)
cond.notifier.register(cond.notifier.ANY, on_conductor_event)
# Run forever, and kill -9 or ctrl-c me...
@@ -166,8 +173,9 @@ def run_poster():
with contextlib.closing(persist_backend):
with contextlib.closing(persist_backend.get_connection()) as conn:
conn.upgrade()
job_backend = job_backends.fetch(my_name, JB_CONF,
persistence=persist_backend)
job_backend = job_backends.fetch(
my_name, JB_CONF, persistence=persist_backend
)
job_backend.connect()
with contextlib.closing(job_backend):
# Create information in the persistence backend about the
@@ -175,14 +183,19 @@ def run_poster():
# can be called to create the tasks that the work unit needs
# to be done.
lb = models.LogBook("post-from-%s" % my_name)
fd = models.FlowDetail("song-from-%s" % my_name,
uuidutils.generate_uuid())
fd = models.FlowDetail(
"song-from-%s" % my_name, uuidutils.generate_uuid()
)
lb.add(fd)
with contextlib.closing(persist_backend.get_connection()) as conn:
conn.save_logbook(lb)
engines.save_factory_details(fd, make_bottles,
[HOW_MANY_BOTTLES], {},
backend=persist_backend)
engines.save_factory_details(
fd,
make_bottles,
[HOW_MANY_BOTTLES],
{},
backend=persist_backend,
)
# Post, and be done with it!
jb = job_backend.post("song-from-%s" % my_name, book=lb)
print("Posted: %s" % jb)

11
taskflow/examples/alphabet_soup.py
View File

@@ -23,9 +23,9 @@ import time
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)
@@ -74,8 +74,9 @@ print("Constructing...")
soup = linear_flow.Flow("alphabet-soup")
for letter in string.ascii_lowercase:
abc = AlphabetTask(letter)
abc.notifier.register(task.EVENT_UPDATE_PROGRESS,
functools.partial(progress_printer, abc))
abc.notifier.register(
task.EVENT_UPDATE_PROGRESS, functools.partial(progress_printer, abc)
)
soup.add(abc)
try:
print("Loading...")

32
taskflow/examples/build_a_car.py
View File

@@ -19,9 +19,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
@@ -64,6 +64,7 @@ def build_wheels():
# These just return true to indiciate success, they would in the real work
# do more than just that.
def install_engine(frame, engine):
return True
@@ -130,15 +131,22 @@ flow = lf.Flow("make-auto").add(
task.FunctorTask(install_engine, provides='engine_installed'),
task.FunctorTask(install_doors, provides='doors_installed'),
task.FunctorTask(install_windows, provides='windows_installed'),
task.FunctorTask(install_wheels, provides='wheels_installed')),
task.FunctorTask(verify, requires=['frame',
'engine',
'doors',
'wheels',
'engine_installed',
'doors_installed',
'windows_installed',
'wheels_installed']))
task.FunctorTask(install_wheels, provides='wheels_installed'),
),
task.FunctorTask(
verify,
requires=[
'frame',
'engine',
'doors',
'wheels',
'engine_installed',
'doors_installed',
'windows_installed',
'wheels_installed',
],
),
)
# This dictionary will be provided to the tasks as a specification for what
# the tasks should produce, in this example this specification will influence

29
taskflow/examples/buildsystem.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import taskflow.engines
@@ -40,17 +40,18 @@ import example_utils as eu # noqa
class CompileTask(task.Task):
"""Pretends to take a source and make object file."""
default_provides = 'object_filename'
def execute(self, source_filename):
object_filename = '%s.o' % os.path.splitext(source_filename)[0]
print('Compiling %s into %s'
% (source_filename, object_filename))
print('Compiling %s into %s' % (source_filename, object_filename))
return object_filename
class LinkTask(task.Task):
"""Pretends to link executable form several object files."""
default_provides = 'executable'
def __init__(self, executable_path, *args, **kwargs):
@@ -59,14 +60,16 @@ class LinkTask(task.Task):
def execute(self, **kwargs):
object_filenames = list(kwargs.values())
print('Linking executable %s from files %s'
% (self._executable_path,
', '.join(object_filenames)))
print(
'Linking executable %s from files %s'
% (self._executable_path, ', '.join(object_filenames))
)
return self._executable_path
class BuildDocsTask(task.Task):
"""Pretends to build docs from sources."""
default_provides = 'docs'
def execute(self, **kwargs):
@@ -84,9 +87,13 @@ def make_flow_and_store(source_files, executable_only=False):
object_stored = '%s-object' % source
store[source_stored] = source
object_targets.append(object_stored)
flow.add(CompileTask(name='compile-%s' % source,
rebind={'source_filename': source_stored},
provides=object_stored))
flow.add(
CompileTask(
name='compile-%s' % source,
rebind={'source_filename': source_stored},
provides=object_stored,
)
)
flow.add(BuildDocsTask(requires=list(store.keys())))
# Try this to see executable_only switch broken:

12
taskflow/examples/calculate_in_parallel.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import taskflow.engines
@@ -65,8 +65,7 @@ flow = lf.Flow('root').add(
# Provide the initial values for other tasks to depend on.
#
# x1 = 2, y1 = 3, x2 = 5, x3 = 8
Provider("provide-adder", 2, 3, 5, 8,
provides=('x1', 'y1', 'x2', 'y2')),
Provider("provide-adder", 2, 3, 5, 8, provides=('x1', 'y1', 'x2', 'y2')),
# Note here that we define the flow that contains the 2 adders to be an
# unordered flow since the order in which these execute does not matter,
# another way to solve this would be to use a graph_flow pattern, which
@@ -85,7 +84,8 @@ flow = lf.Flow('root').add(
Adder(name="add-2", provides='z2', rebind=['x2', 'y2']),
),
# r = z1+z2 = 18
Adder(name="sum-1", provides='r', rebind=['z1', 'z2']))
Adder(name="sum-1", provides='r', rebind=['z1', 'z2']),
)
# The result here will be all results (from all tasks) which is stored in an

12
taskflow/examples/calculate_linear.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import taskflow.engines
@@ -50,7 +50,6 @@ from taskflow import task
# storage backend before your tasks are ran (which accomplishes a similar goal
# in a more uniform manner).
class Provider(task.Task):
def __init__(self, name, *args, **kwargs):
super().__init__(name=name, **kwargs)
self._provide = args
@@ -77,8 +76,7 @@ class Adder(task.Task):
# this function needs to undo if some later operation fails.
class Multiplier(task.Task):
def __init__(self, name, multiplier, provides=None, rebind=None):
super().__init__(name=name, provides=provides,
rebind=rebind)
super().__init__(name=name, provides=provides, rebind=rebind)
self._multiplier = multiplier
def execute(self, z):
@@ -104,7 +102,7 @@ flow = lf.Flow('root').add(
# bound to the 'z' variable provided from the above 'provider' object but
# instead the 'z' argument will be taken from the 'a' variable provided
# by the second add-2 listed above.
Multiplier("multi", 3, provides='r', rebind={'z': 'a'})
Multiplier("multi", 3, provides='r', rebind={'z': 'a'}),
)
# The result here will be all results (from all tasks) which is stored in an

6
taskflow/examples/create_parallel_volume.py
View File

@@ -21,9 +21,9 @@ import time
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from oslo_utils import reflection

9
taskflow/examples/delayed_return.py
View File

@@ -21,9 +21,9 @@ from concurrent import futures
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)
@@ -45,7 +45,8 @@ class PokeFutureListener(base.Listener):
super().__init__(
engine,
task_listen_for=(notifier.Notifier.ANY,),
flow_listen_for=[])
flow_listen_for=[],
)
self._future = future
self._task_name = task_name

57
taskflow/examples/distance_calculator.py
View File

@@ -17,9 +17,9 @@ import math
import os
import sys
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow import engines
@@ -60,24 +60,39 @@ if __name__ == '__main__':
any_distance = linear_flow.Flow("origin").add(DistanceTask())
results = engines.run(any_distance)
print(results)
print("{} is near-enough to {}: {}".format(
results['distance'], 0.0, is_near(results['distance'], 0.0)))
print(
"{} is near-enough to {}: {}".format(
results['distance'], 0.0, is_near(results['distance'], 0.0)
)
)
results = engines.run(any_distance, store={'a': Point(1, 1)})
print(results)
print("{} is near-enough to {}: {}".format(
results['distance'], 1.4142, is_near(results['distance'], 1.4142)))
print(
"{} is near-enough to {}: {}".format(
results['distance'], 1.4142, is_near(results['distance'], 1.4142)
)
)
results = engines.run(any_distance, store={'a': Point(10, 10)})
print(results)
print("{} is near-enough to {}: {}".format(
results['distance'], 14.14199, is_near(results['distance'], 14.14199)))
print(
"{} is near-enough to {}: {}".format(
results['distance'],
14.14199,
is_near(results['distance'], 14.14199),
)
)
results = engines.run(any_distance,
store={'a': Point(5, 5), 'b': Point(10, 10)})
results = engines.run(
any_distance, store={'a': Point(5, 5), 'b': Point(10, 10)}
)
print(results)
print("{} is near-enough to {}: {}".format(
results['distance'], 7.07106, is_near(results['distance'], 7.07106)))
print(
"{} is near-enough to {}: {}".format(
results['distance'], 7.07106, is_near(results['distance'], 7.07106)
)
)
# For this we use the ability to override at task creation time the
# optional arguments so that we don't need to continue to send them
@@ -88,10 +103,18 @@ if __name__ == '__main__':
ten_distance.add(DistanceTask(inject={'a': Point(10, 10)}))
results = engines.run(ten_distance, store={'b': Point(10, 10)})
print(results)
print("{} is near-enough to {}: {}".format(
results['distance'], 0.0, is_near(results['distance'], 0.0)))
print(
"{} is near-enough to {}: {}".format(
results['distance'], 0.0, is_near(results['distance'], 0.0)
)
)
results = engines.run(ten_distance)
print(results)
print("{} is near-enough to {}: {}".format(
results['distance'], 14.14199, is_near(results['distance'], 14.14199)))
print(
"{} is near-enough to {}: {}".format(
results['distance'],
14.14199,
is_near(results['distance'], 14.14199),
)
)

7
taskflow/examples/dump_memory_backend.py
View File

@@ -19,9 +19,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)
@@ -39,6 +39,7 @@ class PrintTask(task.Task):
def execute(self):
print("Running '%s'" % self.name)
# Make a little flow and run it...
f = lf.Flow('root')
for alpha in ['a', 'b', 'c']:

6
taskflow/examples/echo_listener.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.DEBUG)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow import engines

8
taskflow/examples/example_utils.py
View File

@@ -28,6 +28,7 @@ LOG = logging.getLogger(__name__)
try:
import sqlalchemy as _sa # noqa
SQLALCHEMY_AVAILABLE = True
except ImportError:
SQLALCHEMY_AVAILABLE = False
@@ -93,8 +94,11 @@ def get_backend(backend_uri=None):
if not tmp_dir:
tmp_dir = tempfile.mkdtemp()
backend_uri = "file:///%s" % tmp_dir
LOG.exception("Falling back to file backend using temporary"
" directory located at: %s", tmp_dir)
LOG.exception(
"Falling back to file backend using temporary"
" directory located at: %s",
tmp_dir,
)
backend = backends.fetch(_make_conf(backend_uri))
else:
raise e

13
taskflow/examples/fake_billing.py
View File

@@ -20,9 +20,9 @@ import time
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from oslo_utils import uuidutils
@@ -130,8 +130,11 @@ class ActivateDriver(task.Task):
# that the url sending helper class uses. This allows the task progress
# to be tied to the url sending progress, which is very useful for
# downstream systems to be aware of what a task is doing at any time.
url_sender.send(self._url, json.dumps(parsed_request),
status_cb=self.update_progress)
url_sender.send(
self._url,
json.dumps(parsed_request),
status_cb=self.update_progress,
)
return self._url
def update_progress(self, progress, **kwargs):

22
taskflow/examples/graph_flow.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import taskflow.engines
@@ -46,7 +46,6 @@ from taskflow import task
class Adder(task.Task):
def execute(self, x, y):
return x + y
@@ -56,7 +55,7 @@ flow = gf.Flow('root').add(
# x2 = y3+y4 = 12
Adder("add2", provides='x2', rebind=['y3', 'y4']),
# x1 = y1+y2 = 4
Adder("add1", provides='x1', rebind=['y1', 'y2'])
Adder("add1", provides='x1', rebind=['y1', 'y2']),
),
# x5 = x1+x3 = 20
Adder("add5", provides='x5', rebind=['x1', 'x3']),
@@ -67,7 +66,8 @@ flow = gf.Flow('root').add(
# x6 = x5+x4 = 41
Adder("add6", provides='x6', rebind=['x5', 'x4']),
# x7 = x6+x6 = 82
Adder("add7", provides='x7', rebind=['x6', 'x6']))
Adder("add7", provides='x7', rebind=['x6', 'x6']),
)
# Provide the initial variable inputs using a storage dictionary.
store = {
@@ -90,21 +90,19 @@ expected = [
('x7', 82),
]
result = taskflow.engines.run(
flow, engine='serial', store=store)
result = taskflow.engines.run(flow, engine='serial', store=store)
print("Single threaded engine result %s" % result)
for (name, value) in expected:
for name, value in expected:
actual = result.get(name)
if actual != value:
sys.stderr.write(f"{actual} != {value}\n")
unexpected += 1
result = taskflow.engines.run(
flow, engine='parallel', store=store)
result = taskflow.engines.run(flow, engine='parallel', store=store)
print("Multi threaded engine result %s" % result)
for (name, value) in expected:
for name, value in expected:
actual = result.get(name)
if actual != value:
sys.stderr.write(f"{actual} != {value}\n")

57
taskflow/examples/hello_world.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow import engines
@@ -34,6 +34,7 @@ from taskflow import task
# engines using different styles of execution (all can be used to run in
# parallel if a workflow is provided that is parallelizable).
class PrinterTask(task.Task):
def __init__(self, name, show_name=True, inject=None):
super().__init__(name, inject=inject)
@@ -55,26 +56,33 @@ song = lf.Flow("beats")
# singing at once of course!
hi_chorus = uf.Flow('hello')
world_chorus = uf.Flow('world')
for (name, hello, world) in [('bob', 'hello', 'world'),
('joe', 'hellooo', 'worllllld'),
('sue', "helloooooo!", 'wooorllld!')]:
hi_chorus.add(PrinterTask("%s@hello" % name,
# This will show up to the execute() method of
# the task as the argument named 'output' (which
# will allow us to print the character we want).
inject={'output': hello}))
world_chorus.add(PrinterTask("%s@world" % name,
inject={'output': world}))
for name, hello, world in [
('bob', 'hello', 'world'),
('joe', 'hellooo', 'worllllld'),
('sue', "helloooooo!", 'wooorllld!'),
]:
hi_chorus.add(
PrinterTask(
"%s@hello" % name,
# This will show up to the execute() method of
# the task as the argument named 'output' (which
# will allow us to print the character we want).
inject={'output': hello},
)
)
world_chorus.add(PrinterTask("%s@world" % name, inject={'output': world}))
# The composition starts with the conductor and then runs in sequence with
# the chorus running in parallel, but no matter what the 'hello' chorus must
# always run before the 'world' chorus (otherwise the world will fall apart).
song.add(PrinterTask("conductor@begin",
show_name=False, inject={'output': "*ding*"}),
hi_chorus,
world_chorus,
PrinterTask("conductor@end",
show_name=False, inject={'output': "*dong*"}))
song.add(
PrinterTask(
"conductor@begin", show_name=False, inject={'output': "*ding*"}
),
hi_chorus,
world_chorus,
PrinterTask("conductor@end", show_name=False, inject={'output': "*dong*"}),
)
# Run in parallel using eventlet green threads...
try:
@@ -84,22 +92,21 @@ except ImportError:
pass
else:
print("-- Running in parallel using eventlet --")
e = engines.load(song, executor='greenthreaded', engine='parallel',
max_workers=1)
e = engines.load(
song, executor='greenthreaded', engine='parallel', max_workers=1
)
e.run()
# Run in parallel using real threads...
print("-- Running in parallel using threads --")
e = engines.load(song, executor='threaded', engine='parallel',
max_workers=1)
e = engines.load(song, executor='threaded', engine='parallel', max_workers=1)
e.run()
# Run in parallel using external processes...
print("-- Running in parallel using processes --")
e = engines.load(song, executor='processes', engine='parallel',
max_workers=1)
e = engines.load(song, executor='processes', engine='parallel', max_workers=1)
e.run()

17
taskflow/examples/jobboard_produce_consume_colors.py
View File

@@ -23,9 +23,9 @@ import time
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow import exceptions as excp
@@ -121,10 +121,12 @@ def worker(ident, client, consumed):
abandoned_jobs += 1
safe_print(name, "'%s' [abandoned]" % (job))
time.sleep(WORKER_DELAY)
safe_print(name,
"finished (claimed %s jobs, consumed %s jobs,"
" abandoned %s jobs)" % (claimed_jobs, consumed_jobs,
abandoned_jobs), prefix=">>>")
safe_print(
name,
"finished (claimed %s jobs, consumed %s jobs,"
" abandoned %s jobs)" % (claimed_jobs, consumed_jobs, abandoned_jobs),
prefix=">>>",
)
def producer(ident, client):
@@ -149,6 +151,7 @@ def main():
# TODO(harlowja): Hack to make eventlet work right, remove when the
# following is fixed: https://github.com/eventlet/eventlet/issues/230
from taskflow.utils import eventlet_utils as _eu # noqa
try:
import eventlet as _eventlet # noqa
except ImportError:

6
taskflow/examples/parallel_table_multiply.py
View File

@@ -20,9 +20,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import futurist

9
taskflow/examples/persistence_example.py
View File

@@ -21,9 +21,9 @@ import traceback
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)
@@ -95,8 +95,7 @@ with eu.get_backend(backend_uri) as backend:
flow = make_flow(blowup=blowup)
eu.print_wrapped("Running")
try:
eng = engines.load(flow, engine='serial',
backend=backend, book=book)
eng = engines.load(flow, engine='serial', backend=backend, book=book)
eng.run()
if not blowup:
eu.rm_path(persist_path)

28
taskflow/examples/pseudo_scoping.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import taskflow.engines
@@ -43,12 +43,7 @@ from taskflow import task
# his or her phone number from phone book and call.
PHONE_BOOK = {
'jim': '444',
'joe': '555',
'iv_m': '666',
'josh': '777'
}
PHONE_BOOK = {'jim': '444', 'joe': '555', 'iv_m': '666', 'josh': '777'}
class FetchNumberTask(task.Task):
@@ -67,11 +62,10 @@ class CallTask(task.Task):
def execute(self, person, number):
print(f'Calling {person} {number}.')
# This is how it works for one person:
simple_flow = lf.Flow('simple one').add(
FetchNumberTask(),
CallTask())
simple_flow = lf.Flow('simple one').add(FetchNumberTask(), CallTask())
print('Running simple flow:')
taskflow.engines.run(simple_flow, store={'person': 'Josh'})
@@ -85,11 +79,10 @@ def subflow_factory(prefix):
return f'{prefix}-{what}'
return lf.Flow(pr('flow')).add(
FetchNumberTask(pr('fetch'),
provides=pr('number'),
rebind=[pr('person')]),
CallTask(pr('call'),
rebind=[pr('person'), pr('number')])
FetchNumberTask(
pr('fetch'), provides=pr('number'), rebind=[pr('person')]
),
CallTask(pr('call'), rebind=[pr('person'), pr('number')]),
)
@@ -107,5 +100,6 @@ def call_them_all():
flow.add(subflow_factory(prefix))
taskflow.engines.run(flow, store=persons)
print('\nCalling many people using prefixed factory:')
call_them_all()

31
taskflow/examples/resume_from_backend.py
View File

@@ -20,9 +20,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)
@@ -63,8 +63,9 @@ def print_task_states(flowdetail, msg):
print(f"Flow '{flowdetail.name}' state: {flowdetail.state}")
# Sort by these so that our test validation doesn't get confused by the
# order in which the items in the flow detail can be in.
items = sorted((td.name, td.version, td.state, td.results)
for td in flowdetail)
items = sorted(
(td.name, td.version, td.state, td.results) for td in flowdetail
)
for item in items:
print(" %s==%s: %s, result=%s" % item)
@@ -94,17 +95,18 @@ def flow_factory():
return lf.Flow('resume from backend example').add(
TestTask(name='first'),
InterruptTask(name='boom'),
TestTask(name='second'))
TestTask(name='second'),
)
# INITIALIZE PERSISTENCE ####################################
with eu.get_backend() as backend:
# Create a place where the persistence information will be stored.
book = models.LogBook("example")
flow_detail = models.FlowDetail("resume from backend example",
uuid=uuidutils.generate_uuid())
flow_detail = models.FlowDetail(
"resume from backend example", uuid=uuidutils.generate_uuid()
)
book.add(flow_detail)
with contextlib.closing(backend.get_connection()) as conn:
conn.save_logbook(book)
@@ -112,8 +114,9 @@ with eu.get_backend() as backend:
# CREATE AND RUN THE FLOW: FIRST ATTEMPT ####################
flow = flow_factory()
engine = taskflow.engines.load(flow, flow_detail=flow_detail,
book=book, backend=backend)
engine = taskflow.engines.load(
flow, flow_detail=flow_detail, book=book, backend=backend
)
print_task_states(flow_detail, "At the beginning, there is no state")
eu.print_wrapped("Running")
@@ -135,8 +138,8 @@ with eu.get_backend() as backend:
# running the above flow crashes).
flow2 = flow_factory()
flow_detail_2 = find_flow_detail(backend, book.uuid, flow_detail.uuid)
engine2 = taskflow.engines.load(flow2,
flow_detail=flow_detail_2,
backend=backend, book=book)
engine2 = taskflow.engines.load(
flow2, flow_detail=flow_detail_2, backend=backend, book=book
)
engine2.run()
print_task_states(flow_detail_2, "At the end")

12
taskflow/examples/resume_many_flows.py
View File

@@ -42,9 +42,9 @@ def _exec(cmd, add_env=None):
env = os.environ.copy()
env.update(add_env)
proc = subprocess.Popen(cmd, env=env, stdin=None,
stdout=subprocess.PIPE,
stderr=sys.stderr)
proc = subprocess.Popen(
cmd, env=env, stdin=None, stdout=subprocess.PIPE, stderr=sys.stderr
)
stdout, _stderr = proc.communicate()
rc = proc.returncode
@@ -54,8 +54,9 @@ def _exec(cmd, add_env=None):
def _path_to(name):
return os.path.abspath(os.path.join(os.path.dirname(__file__),
'resume_many_flows', name))
return os.path.abspath(
os.path.join(os.path.dirname(__file__), 'resume_many_flows', name)
)
def main():
@@ -87,5 +88,6 @@ def main():
if tmp_path:
example_utils.rm_path(tmp_path)
if __name__ == '__main__':
main()

3
taskflow/examples/resume_many_flows/my_flows.py
View File

@@ -38,4 +38,5 @@ def flow_factory():
return lf.Flow('example').add(
TestTask(name='first'),
UnfortunateTask(name='boom'),
TestTask(name='second'))
TestTask(name='second'),
)

8
taskflow/examples/resume_many_flows/resume_all.py
View File

@@ -20,7 +20,8 @@ logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(
os.path.join(self_dir, os.pardir, os.pardir, os.pardir))
os.path.join(self_dir, os.pardir, os.pardir, os.pardir)
)
example_dir = os.path.abspath(os.path.join(self_dir, os.pardir))
sys.path.insert(0, top_dir)
@@ -38,8 +39,9 @@ FINISHED_STATES = (states.SUCCESS, states.FAILURE, states.REVERTED)
def resume(flowdetail, backend):
print(f'Resuming flow {flowdetail.name} {flowdetail.uuid}')
engine = taskflow.engines.load_from_detail(flow_detail=flowdetail,
backend=backend)
engine = taskflow.engines.load_from_detail(
flow_detail=flowdetail, backend=backend
)
engine.run()

15
taskflow/examples/resume_many_flows/run_flow.py
View File

@@ -20,7 +20,8 @@ logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(
os.path.join(self_dir, os.pardir, os.pardir, os.pardir))
os.path.join(self_dir, os.pardir, os.pardir, os.pardir)
)
example_dir = os.path.abspath(os.path.join(self_dir, os.pardir))
sys.path.insert(0, top_dir)
@@ -34,8 +35,12 @@ import my_flows # noqa
with example_utils.get_backend() as backend:
engine = taskflow.engines.load_from_factory(my_flows.flow_factory,
backend=backend)
print('Running flow {} {}'.format(engine.storage.flow_name,
engine.storage.flow_uuid))
engine = taskflow.engines.load_from_factory(
my_flows.flow_factory, backend=backend
)
print(
'Running flow {} {}'.format(
engine.storage.flow_name, engine.storage.flow_uuid
)
)
engine.run()

55
taskflow/examples/resume_vm_boot.py
View File

@@ -23,9 +23,9 @@ import time
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)
@@ -59,6 +59,7 @@ def slow_down(how_long=0.5):
class PrintText(task.Task):
"""Just inserts some text print outs in a workflow."""
def __init__(self, print_what, no_slow=False):
content_hash = hashlib.md5(print_what.encode('utf-8')).hexdigest()[0:8]
super().__init__(name="Print: %s" % (content_hash))
@@ -75,6 +76,7 @@ class PrintText(task.Task):
class DefineVMSpec(task.Task):
"""Defines a vm specification to be."""
def __init__(self, name):
super().__init__(provides='vm_spec', name=name)
@@ -90,6 +92,7 @@ class DefineVMSpec(task.Task):
class LocateImages(task.Task):
"""Locates where the vm images are."""
def __init__(self, name):
super().__init__(provides='image_locations', name=name)
@@ -103,9 +106,9 @@ class LocateImages(task.Task):
class DownloadImages(task.Task):
"""Downloads all the vm images."""
def __init__(self, name):
super().__init__(provides='download_paths',
name=name)
super().__init__(provides='download_paths', name=name)
def execute(self, image_locations):
for src, loc in image_locations.items():
@@ -116,14 +119,14 @@ class DownloadImages(task.Task):
class CreateNetworkTpl(task.Task):
"""Generates the network settings file to be placed in the images."""
SYSCONFIG_CONTENTS = """DEVICE=eth%s
BOOTPROTO=static
IPADDR=%s
ONBOOT=yes"""
def __init__(self, name):
super().__init__(provides='network_settings',
name=name)
super().__init__(provides='network_settings', name=name)
def execute(self, ips):
settings = []
@@ -134,6 +137,7 @@ ONBOOT=yes"""
class AllocateIP(task.Task):
"""Allocates the ips for the given vm."""
def __init__(self, name):
super().__init__(provides='ips', name=name)
@@ -146,13 +150,15 @@ class AllocateIP(task.Task):
class WriteNetworkSettings(task.Task):
"""Writes all the network settings into the downloaded images."""
def execute(self, download_paths, network_settings):
for j, path in enumerate(download_paths):
with slow_down(1):
print(f"Mounting {path} to /tmp/{j}")
for i, setting in enumerate(network_settings):
filename = ("/tmp/etc/sysconfig/network-scripts/"
"ifcfg-eth%s" % (i))
filename = "/tmp/etc/sysconfig/network-scripts/ifcfg-eth%s" % (
i
)
with slow_down(1):
print("Writing to %s" % (filename))
print(setting)
@@ -160,6 +166,7 @@ class WriteNetworkSettings(task.Task):
class BootVM(task.Task):
"""Fires off the vm boot operation."""
def execute(self, vm_spec):
print("Starting vm!")
with slow_down(1):
@@ -168,6 +175,7 @@ class BootVM(task.Task):
class AllocateVolumes(task.Task):
"""Allocates the volumes for the vm."""
def execute(self, vm_spec):
volumes = []
for i in range(0, vm_spec['volumes']):
@@ -179,6 +187,7 @@ class AllocateVolumes(task.Task):
class FormatVolumes(task.Task):
"""Formats the volumes for the vm."""
def execute(self, volumes):
for v in volumes:
print("Formatting volume %s" % v)
@@ -215,14 +224,15 @@ def create_flow():
),
# Ya it worked!
PrintText("Finished vm create.", no_slow=True),
PrintText("Instance is running!", no_slow=True))
PrintText("Instance is running!", no_slow=True),
)
return flow
eu.print_wrapped("Initializing")
# Setup the persistence & resumption layer.
with eu.get_backend() as backend:
# Try to find a previously passed in tracking id...
try:
book_id, flow_id = sys.argv[2].split("+", 1)
@@ -256,17 +266,24 @@ with eu.get_backend() as backend:
book = models.LogBook("vm-boot")
with contextlib.closing(backend.get_connection()) as conn:
conn.save_logbook(book)
engine = engines.load_from_factory(create_flow,
backend=backend, book=book,
engine='parallel',
executor=executor)
print("!! Your tracking id is: '{}+{}'".format(
book.uuid, engine.storage.flow_uuid))
engine = engines.load_from_factory(
create_flow,
backend=backend,
book=book,
engine='parallel',
executor=executor,
)
print(
"!! Your tracking id is: '{}+{}'".format(
book.uuid, engine.storage.flow_uuid
)
)
print("!! Please submit this on later runs for tracking purposes")
else:
# Attempt to load from a previously partially completed flow.
engine = engines.load_from_detail(flow_detail, backend=backend,
engine='parallel', executor=executor)
engine = engines.load_from_detail(
flow_detail, backend=backend, engine='parallel', executor=executor
)
# Make me my vm please!
eu.print_wrapped('Running')

32
taskflow/examples/resume_volume_create.py
View File

@@ -23,9 +23,9 @@ import time
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)
@@ -90,10 +90,12 @@ class CreateSpecForVolumes(task.Task):
def execute(self):
volumes = []
for i in range(0, random.randint(1, 10)):
volumes.append({
'type': 'disk',
'location': "/dev/vda%s" % (i + 1),
})
volumes.append(
{
'type': 'disk',
'location': "/dev/vda%s" % (i + 1),
}
)
return volumes
@@ -115,7 +117,8 @@ flow = lf.Flow("root").add(
PrintText("I need a nap, it took me a while to build those specs."),
PrepareVolumes(),
),
PrintText("Finished volume create", no_slow=True))
PrintText("Finished volume create", no_slow=True),
)
# Setup the persistence & resumption layer.
with example_utils.get_backend() as backend:
@@ -139,16 +142,19 @@ with example_utils.get_backend() as backend:
book.add(flow_detail)
with contextlib.closing(backend.get_connection()) as conn:
conn.save_logbook(book)
print("!! Your tracking id is: '{}+{}'".format(book.uuid,
flow_detail.uuid))
print(
"!! Your tracking id is: '{}+{}'".format(
book.uuid, flow_detail.uuid
)
)
print("!! Please submit this on later runs for tracking purposes")
else:
flow_detail = find_flow_detail(backend, book_id, flow_id)
# Load and run.
engine = engines.load(flow,
flow_detail=flow_detail,
backend=backend, engine='serial')
engine = engines.load(
flow, flow_detail=flow_detail, backend=backend, engine='serial'
)
engine.run()
# How to use.

16
taskflow/examples/retry_flow.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import taskflow.engines
@@ -52,10 +52,12 @@ class CallJim(task.Task):
# Create your flow and associated tasks (the work to be done).
flow = lf.Flow('retrying-linear',
retry=retry.ParameterizedForEach(
rebind=['phone_directory'],
provides='jim_number')).add(CallJim())
flow = lf.Flow(
'retrying-linear',
retry=retry.ParameterizedForEach(
rebind=['phone_directory'], provides='jim_number'
),
).add(CallJim())
# Now run that flow using the provided initial data (store below).
taskflow.engines.run(flow, store={'phone_directory': [333, 444, 555, 666]})

18
taskflow/examples/reverting_linear.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import taskflow.engines
@@ -61,17 +61,13 @@ class CallSuzzie(task.Task):
# Create your flow and associated tasks (the work to be done).
flow = lf.Flow('simple-linear').add(
CallJim(),
CallJoe(),
CallSuzzie()
)
flow = lf.Flow('simple-linear').add(CallJim(), CallJoe(), CallSuzzie())
try:
# Now run that flow using the provided initial data (store below).
taskflow.engines.run(flow, store=dict(joe_number=444,
jim_number=555,
suzzie_number=666))
taskflow.engines.run(
flow, store=dict(joe_number=444, jim_number=555, suzzie_number=666)
)
except Exception as e:
# NOTE(harlowja): This exception will be the exception that came out of the
# 'CallSuzzie' task instead of a different exception, this is useful since

23
taskflow/examples/run_by_iter.py
View File

@@ -19,9 +19,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)
@@ -51,12 +51,19 @@ def make_alphabet_flow(i):
while ord(curr_value) <= ord(end_value):
next_value = chr(ord(curr_value) + 1)
if curr_value != end_value:
f.add(EchoTask(name="echoer_%s" % curr_value,
rebind={'value': curr_value},
provides=next_value))
f.add(
EchoTask(
name="echoer_%s" % curr_value,
rebind={'value': curr_value},
provides=next_value,
)
)
else:
f.add(EchoTask(name="echoer_%s" % curr_value,
rebind={'value': curr_value}))
f.add(
EchoTask(
name="echoer_%s" % curr_value, rebind={'value': curr_value}
)
)
curr_value = next_value
return f

6
taskflow/examples/run_by_iter_enumerate.py
View File

@@ -19,9 +19,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)

6
taskflow/examples/share_engine_thread.py
View File

@@ -20,9 +20,9 @@ import time
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import futurist

14
taskflow/examples/simple_linear.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import taskflow.engines
@@ -54,11 +54,7 @@ class CallJoe(task.Task):
# Create your flow and associated tasks (the work to be done).
flow = lf.Flow('simple-linear').add(
CallJim(),
CallJoe()
)
flow = lf.Flow('simple-linear').add(CallJim(), CallJoe())
# Now run that flow using the provided initial data (store below).
taskflow.engines.run(flow, store=dict(joe_number=444,
jim_number=555))
taskflow.engines.run(flow, store=dict(joe_number=444, jim_number=555))

21
taskflow/examples/simple_linear_listening.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
import taskflow.engines
@@ -82,12 +82,15 @@ flow.add(task.FunctorTask(execute=call_jim))
flow.add(task.FunctorTask(execute=call_joe))
# Now load (but do not run) the flow using the provided initial data.
engine = taskflow.engines.load(flow, store={
'context': {
"joe_number": 444,
"jim_number": 555,
}
})
engine = taskflow.engines.load(
flow,
store={
'context': {
"joe_number": 444,
"jim_number": 555,
}
},
)
# This is where we attach our callback functions to the 2 different
# notification objects that an engine exposes. The usage of a ANY (kleene star)

6
taskflow/examples/simple_linear_pass.py
View File

@@ -19,9 +19,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)

18
taskflow/examples/simple_map_reduce.py
View File

@@ -19,9 +19,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)
@@ -47,7 +47,7 @@ class TotalReducer(task.Task):
def execute(self, *args, **kwargs):
# Reduces all mapped summed outputs into a single value.
total = 0
for (k, v) in kwargs.items():
for k, v in kwargs.items():
# If any other kwargs was passed in, we don't want to use those
# in the calculation of the total...
if k.startswith('reduction_'):
@@ -88,9 +88,13 @@ for i, chunk in enumerate(chunk_iter(CHUNK_SIZE, UPPER_BOUND)):
# The reducer uses all of the outputs of the mappers, so it needs
# to be recorded that it needs access to them (under a specific name).
provided.append("reduction_%s" % i)
mappers.add(SumMapper(name=mapper_name,
rebind={'inputs': mapper_name},
provides=provided[-1]))
mappers.add(
SumMapper(
name=mapper_name,
rebind={'inputs': mapper_name},
provides=provided[-1],
)
)
w.add(mappers)
# The reducer will run last (after all the mappers).

17
taskflow/examples/switch_graph_flow.py
View File

@@ -18,9 +18,9 @@ import sys
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow import engines
@@ -56,8 +56,10 @@ print("---------")
print("After run")
print("---------")
backend = e.storage.backend
entries = [os.path.join(backend.memory.root_path, child)
for child in backend.memory.ls(backend.memory.root_path)]
entries = [
os.path.join(backend.memory.root_path, child)
for child in backend.memory.ls(backend.memory.root_path)
]
while entries:
path = entries.pop()
value = backend.memory[path]
@@ -65,5 +67,6 @@ while entries:
print(f"{path} -> {value}")
else:
print("%s" % (path))
entries.extend(os.path.join(path, child)
for child in backend.memory.ls(path))
entries.extend(
os.path.join(path, child) for child in backend.memory.ls(path)
)

6
taskflow/examples/timing_listener.py
View File

@@ -20,9 +20,9 @@ import time
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow import engines

39
taskflow/examples/tox_conductor.py
View File

@@ -25,9 +25,9 @@ import time
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from oslo_utils import timeutils
@@ -128,7 +128,7 @@ def create_review_workflow():
f.add(
MakeTempDir(name="maker"),
RunReview(name="runner"),
CleanResources(name="cleaner")
CleanResources(name="cleaner"),
)
return f
@@ -137,14 +137,16 @@ def generate_reviewer(client, saver, name=NAME):
"""Creates a review producer thread with the given name prefix."""
real_name = "%s_reviewer" % name
no_more = threading.Event()
jb = boards.fetch(real_name, JOBBOARD_CONF,
client=client, persistence=saver)
jb = boards.fetch(
real_name, JOBBOARD_CONF, client=client, persistence=saver
)
def make_save_book(saver, review_id):
# Record what we want to happen (sometime in the future).
book = models.LogBook("book_%s" % review_id)
detail = models.FlowDetail("flow_%s" % review_id,
uuidutils.generate_uuid())
detail = models.FlowDetail(
"flow_%s" % review_id, uuidutils.generate_uuid()
)
book.add(detail)
# Associate the factory method we want to be called (in the future)
# with the book, so that the conductor will be able to call into
@@ -157,8 +159,9 @@ def generate_reviewer(client, saver, name=NAME):
# workflow that represents this review).
factory_args = ()
factory_kwargs = {}
engines.save_factory_details(detail, create_review_workflow,
factory_args, factory_kwargs)
engines.save_factory_details(
detail, create_review_workflow, factory_args, factory_kwargs
)
with contextlib.closing(saver.get_connection()) as conn:
conn.save_logbook(book)
return book
@@ -177,9 +180,11 @@ def generate_reviewer(client, saver, name=NAME):
}
job_name = "{}_{}".format(real_name, review['id'])
print("Posting review '%s'" % review['id'])
jb.post(job_name,
book=make_save_book(saver, review['id']),
details=details)
jb.post(
job_name,
book=make_save_book(saver, review['id']),
details=details,
)
time.sleep(REVIEW_CREATION_DELAY)
# Return the unstarted thread, and a callback that can be used
@@ -190,10 +195,10 @@ def generate_reviewer(client, saver, name=NAME):
def generate_conductor(client, saver, name=NAME):
"""Creates a conductor thread with the given name prefix."""
real_name = "%s_conductor" % name
jb = boards.fetch(name, JOBBOARD_CONF,
client=client, persistence=saver)
conductor = conductors.fetch("blocking", real_name, jb,
engine='parallel', wait_timeout=SCAN_DELAY)
jb = boards.fetch(name, JOBBOARD_CONF, client=client, persistence=saver)
conductor = conductors.fetch(
"blocking", real_name, jb, engine='parallel', wait_timeout=SCAN_DELAY
)
def run():
jb.connect()

6
taskflow/examples/wbe_event_sender.py
View File

@@ -18,9 +18,9 @@ import string
import sys
import time
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow import engines

47
taskflow/examples/wbe_mandelbrot.py
View File

@@ -17,9 +17,9 @@ import math
import os
import sys
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow import engines
@@ -118,7 +118,7 @@ def calculate(engine_conf):
'mandelbrot_config': [-2.0, 1.0, -1.0, 1.0, MAX_ITERATIONS],
'image_config': {
'size': IMAGE_SIZE,
}
},
}
# We need the task names to be in the right order so that we can extract
@@ -135,13 +135,16 @@ def calculate(engine_conf):
# Break the calculation up into chunk size pieces.
rows = [i * chunk_size, i * chunk_size + chunk_size]
flow.add(
MandelCalculator(task_name,
# This ensures the storage symbol with name
# 'chunk_name' is sent into the tasks local
# symbol 'chunk'. This is how we give each
# calculator its own correct sequence of rows
# to work on.
rebind={'chunk': chunk_name}))
MandelCalculator(
task_name,
# This ensures the storage symbol with name
# 'chunk_name' is sent into the tasks local
# symbol 'chunk'. This is how we give each
# calculator its own correct sequence of rows
# to work on.
rebind={'chunk': chunk_name},
)
)
store[chunk_name] = rows
task_names.append(task_name)
@@ -161,9 +164,11 @@ def calculate(engine_conf):
def write_image(results, output_filename=None):
print("Gathered %s results that represents a mandelbrot"
" image (using %s chunks that are computed jointly"
" by %s workers)." % (len(results), CHUNK_COUNT, WORKERS))
print(
"Gathered %s results that represents a mandelbrot"
" image (using %s chunks that are computed jointly"
" by %s workers)." % (len(results), CHUNK_COUNT, WORKERS)
)
if not output_filename:
return
@@ -198,12 +203,14 @@ def create_fractal():
# Setup our transport configuration and merge it into the worker and
# engine configuration so that both of those use it correctly.
shared_conf = dict(BASE_SHARED_CONF)
shared_conf.update({
'transport': 'memory',
'transport_options': {
'polling_interval': 0.1,
},
})
shared_conf.update(
{
'transport': 'memory',
'transport_options': {
'polling_interval': 0.1,
},
}
)
if len(sys.argv) >= 2:
output_filename = sys.argv[1]

51
taskflow/examples/wbe_simple_linear.py
View File

@@ -18,9 +18,9 @@ import os
import sys
import tempfile
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
from taskflow import engines
@@ -64,7 +64,7 @@ WORKER_CONF = {
# not want to allow all python code to be executed).
'tasks': [
'taskflow.tests.utils:TaskOneArgOneReturn',
'taskflow.tests.utils:TaskMultiArgOneReturn'
'taskflow.tests.utils:TaskMultiArgOneReturn',
],
}
@@ -72,11 +72,14 @@ WORKER_CONF = {
def run(engine_options):
flow = lf.Flow('simple-linear').add(
utils.TaskOneArgOneReturn(provides='result1'),
utils.TaskMultiArgOneReturn(provides='result2')
utils.TaskMultiArgOneReturn(provides='result2'),
)
eng = engines.load(
flow,
store=dict(x=111, y=222, z=333),
engine='worker-based',
**engine_options,
)
eng = engines.load(flow,
store=dict(x=111, y=222, z=333),
engine='worker-based', **engine_options)
eng.run()
return eng.storage.fetch_all()
@@ -92,22 +95,26 @@ if __name__ == "__main__":
if USE_FILESYSTEM:
worker_count = FILE_WORKERS
tmp_path = tempfile.mkdtemp(prefix='wbe-example-')
shared_conf.update({
'transport': 'filesystem',
'transport_options': {
'data_folder_in': tmp_path,
'data_folder_out': tmp_path,
'polling_interval': 0.1,
},
})
shared_conf.update(
{
'transport': 'filesystem',
'transport_options': {
'data_folder_in': tmp_path,
'data_folder_out': tmp_path,
'polling_interval': 0.1,
},
}
)
else:
worker_count = MEMORY_WORKERS
shared_conf.update({
'transport': 'memory',
'transport_options': {
'polling_interval': 0.1,
},
})
shared_conf.update(
{
'transport': 'memory',
'transport_options': {
'polling_interval': 0.1,
},
}
)
worker_conf = dict(WORKER_CONF)
worker_conf.update(shared_conf)
engine_options = dict(shared_conf)

23
taskflow/examples/wrapped_exception.py
View File

@@ -20,9 +20,9 @@ import time
logging.basicConfig(level=logging.ERROR)
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
top_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
sys.path.insert(0, top_dir)
@@ -84,14 +84,10 @@ def run(**store):
# here and based on those kwargs it will behave in a different manner
# while executing; this allows for the calling code (see below) to show
# different usages of the failure catching and handling mechanism.
flow = uf.Flow('flow').add(
FirstTask(),
SecondTask()
)
flow = uf.Flow('flow').add(FirstTask(), SecondTask())
try:
with utils.wrap_all_failures():
taskflow.engines.run(flow, store=store,
engine='parallel')
taskflow.engines.run(flow, store=store, engine='parallel')
except exceptions.WrappedFailure as ex:
unknown_failures = []
for a_failure in ex:
@@ -106,20 +102,17 @@ def run(**store):
eu.print_wrapped("Raise and catch first exception only")
run(sleep1=0.0, raise1=True,
sleep2=0.0, raise2=False)
run(sleep1=0.0, raise1=True, sleep2=0.0, raise2=False)
# NOTE(imelnikov): in general, sleeping does not guarantee that we'll have both
# task running before one of them fails, but with current implementation this
# works most of times, which is enough for our purposes here (as an example).
eu.print_wrapped("Raise and catch both exceptions")
run(sleep1=1.0, raise1=True,
sleep2=1.0, raise2=True)
run(sleep1=1.0, raise1=True, sleep2=1.0, raise2=True)
eu.print_wrapped("Handle one exception, and re-raise another")
try:
run(sleep1=1.0, raise1=True,
sleep2=1.0, raise2='boom')
run(sleep1=1.0, raise1=True, sleep2=1.0, raise2='boom')
except TypeError as ex:
print("As expected, TypeError is here: %s" % ex)
else:

23
taskflow/exceptions.py
View File

@@ -64,6 +64,7 @@ class TaskFlowException(Exception):
creating a chain of exceptions for versions of python where
this is not yet implemented/supported natively.
"""
def __init__(self, message, cause=None):
super().__init__(message)
self._cause = cause
@@ -84,8 +85,10 @@ class TaskFlowException(Exception):
def pformat(self, indent=2, indent_text=" ", show_root_class=False):
"""Pretty formats a taskflow exception + any connected causes."""
if indent < 0:
raise ValueError("Provided 'indent' must be greater than"
" or equal to zero instead of %s" % indent)
raise ValueError(
"Provided 'indent' must be greater than"
" or equal to zero instead of %s" % indent
)
buf = io.StringIO()
if show_root_class:
buf.write(reflection.get_class_name(self, fully_qualified=False))
@@ -99,8 +102,9 @@ class TaskFlowException(Exception):
buf.write(os.linesep)
if isinstance(next_up, TaskFlowException):
buf.write(indent_text * active_indent)
buf.write(reflection.get_class_name(next_up,
fully_qualified=False))
buf.write(
reflection.get_class_name(next_up, fully_qualified=False)
)
buf.write(": ")
buf.write(next_up._get_message())
else:
@@ -125,18 +129,21 @@ class TaskFlowException(Exception):
# Errors related to storage or operations on storage units.
class StorageFailure(TaskFlowException):
"""Raised when storage backends can not be read/saved/deleted."""
# Conductor related errors.
class ConductorFailure(TaskFlowException):
"""Errors related to conducting activities."""
# Job related errors.
class JobFailure(TaskFlowException):
"""Errors related to jobs or operations on jobs."""
@@ -147,6 +154,7 @@ class UnclaimableJob(JobFailure):
# Engine/ during execution related errors.
class ExecutionFailure(TaskFlowException):
"""Errors related to engine execution."""
@@ -181,8 +189,10 @@ class MissingDependencies(DependencyFailure):
"""
#: Exception message template used when creating an actual message.
MESSAGE_TPL = ("'%(who)s' requires %(requirements)s but no other entity"
" produces said requirements")
MESSAGE_TPL = (
"'%(who)s' requires %(requirements)s but no other entity"
" produces said requirements"
)
METHOD_TPL = "'%(method)s' method on "
@@ -232,6 +242,7 @@ class DisallowedAccess(TaskFlowException):
# Others.
class NotImplementedError(NotImplementedError):
"""Exception for when some functionality really isn't implemented.

73
taskflow/formatters.py
View File

@@ -63,8 +63,13 @@ class FailureFormatter:
states.EXECUTE: (_fetch_predecessor_tree, 'predecessors'),
}
def __init__(self, engine, hide_inputs_outputs_of=(),
mask_inputs_keys=(), mask_outputs_keys=()):
def __init__(
self,
engine,
hide_inputs_outputs_of=(),
mask_inputs_keys=(),
mask_outputs_keys=(),
):
self._hide_inputs_outputs_of = hide_inputs_outputs_of
self._mask_inputs_keys = mask_inputs_keys
self._mask_outputs_keys = mask_outputs_keys
@@ -95,13 +100,17 @@ class FailureFormatter:
atom_name = atom.name
atom_attrs = {}
intention, intention_found = _cached_get(
cache, 'intentions', atom_name, storage.get_atom_intention,
atom_name)
cache,
'intentions',
atom_name,
storage.get_atom_intention,
atom_name,
)
if intention_found:
atom_attrs['intention'] = intention
state, state_found = _cached_get(cache, 'states', atom_name,
storage.get_atom_state,
atom_name)
state, state_found = _cached_get(
cache, 'states', atom_name, storage.get_atom_state, atom_name
)
if state_found:
atom_attrs['state'] = state
if atom_name not in self._hide_inputs_outputs_of:
@@ -109,27 +118,38 @@ class FailureFormatter:
# will be called with the rest of these arguments
# used to populate the cache.
fetch_mapped_args = functools.partial(
storage.fetch_mapped_args, atom.rebind,
atom_name=atom_name, optional_args=atom.optional)
requires, requires_found = _cached_get(cache, 'requires',
atom_name,
fetch_mapped_args)
storage.fetch_mapped_args,
atom.rebind,
atom_name=atom_name,
optional_args=atom.optional,
)
requires, requires_found = _cached_get(
cache, 'requires', atom_name, fetch_mapped_args
)
if requires_found:
atom_attrs['requires'] = self._mask_keys(
requires, self._mask_inputs_keys)
requires, self._mask_inputs_keys
)
provides, provides_found = _cached_get(
cache, 'provides', atom_name,
storage.get_execute_result, atom_name)
cache,
'provides',
atom_name,
storage.get_execute_result,
atom_name,
)
if provides_found:
atom_attrs['provides'] = self._mask_keys(
provides, self._mask_outputs_keys)
provides, self._mask_outputs_keys
)
if atom_attrs:
return f"Atom '{atom_name}' {atom_attrs}"
else:
return "Atom '%s'" % (atom_name)
else:
raise TypeError("Unable to format node, unknown node"
" kind '%s' encountered" % node.metadata['kind'])
raise TypeError(
"Unable to format node, unknown node"
" kind '%s' encountered" % node.metadata['kind']
)
def format(self, fail, atom_matcher):
"""Returns a (exc_info, details) tuple about the failure.
@@ -173,15 +193,20 @@ class FailureFormatter:
builder, kind = self._BUILDERS[atom_intention]
rooted_tree = builder(graph, atom)
child_count = rooted_tree.child_count(only_direct=False)
buff.write_nl(
f'{child_count} {kind} (most recent first):')
buff.write_nl(f'{child_count} {kind} (most recent first):')
formatter = functools.partial(self._format_node, storage, cache)
direct_child_count = rooted_tree.child_count(only_direct=True)
for i, child in enumerate(rooted_tree, 1):
if i == direct_child_count:
buff.write(child.pformat(stringify_node=formatter,
starting_prefix=" "))
buff.write(
child.pformat(
stringify_node=formatter, starting_prefix=" "
)
)
else:
buff.write_nl(child.pformat(stringify_node=formatter,
starting_prefix=" "))
buff.write_nl(
child.pformat(
stringify_node=formatter, starting_prefix=" "
)
)
return (fail.exc_info, buff.getvalue())

11
taskflow/jobs/backends/__init__.py
View File

@@ -51,10 +51,13 @@ def fetch(name, conf, namespace=BACKEND_NAMESPACE, **kwargs):
board, conf = misc.extract_driver_and_conf(conf, 'board')
LOG.debug('Looking for %r jobboard driver in %r', board, namespace)
try:
mgr = driver.DriverManager(namespace, board,
invoke_on_load=True,
invoke_args=(name, conf),
invoke_kwds=kwargs)
mgr = driver.DriverManager(
namespace,
board,
invoke_on_load=True,
invoke_args=(name, conf),
invoke_kwds=kwargs,
)
return mgr.driver
except RuntimeError as e:
raise exc.NotFound("Could not find jobboard %s" % (board), e)

183
taskflow/jobs/backends/impl_etcd.py
View File

@@ -26,6 +26,7 @@ from taskflow.jobs import base
from taskflow import logging
from taskflow import states
from taskflow.utils import misc
if typing.TYPE_CHECKING:
from taskflow.types import entity
@@ -37,13 +38,30 @@ class EtcdJob(base.Job):
board: 'EtcdJobBoard'
def __init__(self, board: 'EtcdJobBoard', name, client, key,
uuid=None, details=None, backend=None,
book=None, book_data=None,
priority=base.JobPriority.NORMAL,
sequence=None, created_on=None):
super().__init__(board, name, uuid=uuid, details=details,
backend=backend, book=book, book_data=book_data)
def __init__(
self,
board: 'EtcdJobBoard',
name,
client,
key,
uuid=None,
details=None,
backend=None,
book=None,
book_data=None,
priority=base.JobPriority.NORMAL,
sequence=None,
created_on=None,
):
super().__init__(
board,
name,
uuid=uuid,
details=details,
backend=backend,
book=book,
book_data=book_data,
)
self._client = client
self._key = key
@@ -79,8 +97,11 @@ class EtcdJob(base.Job):
owner, data = self.board.get_owner_and_data(self)
if not data:
if owner is not None:
LOG.info(f"Owner key was found for job {self.uuid}, "
f"but the key {self.key} is missing")
LOG.info(
"Owner key was found for job %s but the key %s is missing",
self.uuid,
self.key,
)
return states.COMPLETE
if not owner:
return states.UNCLAIMED
@@ -101,8 +122,7 @@ class EtcdJob(base.Job):
if 'lease_id' not in owner_data:
return None
lease_id = owner_data['lease_id']
self._lease = etcd3gw.Lease(id=lease_id,
client=self._client)
self._lease = etcd3gw.Lease(id=lease_id, client=self._client)
return self._lease
def expires_in(self):
@@ -120,7 +140,7 @@ class EtcdJob(base.Job):
if self.lease is None:
return False
ret = self.lease.refresh()
return (ret > 0)
return ret > 0
@property
def root(self):
@@ -134,7 +154,8 @@ class EtcdJob(base.Job):
return self.sequence < other.sequence
else:
ordered = base.JobPriority.reorder(
(self.priority, self), (other.priority, other))
(self.priority, self), (other.priority, other)
)
if ordered[0] is self:
return False
return True
@@ -145,8 +166,11 @@ class EtcdJob(base.Job):
def __eq__(self, other):
if not isinstance(other, EtcdJob):
return NotImplemented
return ((self.root, self.sequence, self.priority) ==
(other.root, other.sequence, other.priority))
return (self.root, self.sequence, self.priority) == (
other.root,
other.sequence,
other.priority,
)
def __ne__(self, other):
return not self.__eq__(other)
@@ -184,6 +208,7 @@ class EtcdJobBoard(base.JobBoard):
.. _etcd: https://etcd.io/
"""
ROOT_PATH = "/taskflow/jobs"
TRASH_PATH = "/taskflow/.trash"
@@ -224,8 +249,10 @@ class EtcdJobBoard(base.JobBoard):
self._persistence = persistence
self._state = self.INIT_STATE
path_elems = [self.ROOT_PATH,
self._conf.get("path", self.DEFAULT_PATH)]
path_elems = [
self.ROOT_PATH,
self._conf.get("path", self.DEFAULT_PATH),
]
self._root_path = self._create_path(*path_elems)
self._job_cache = {}
@@ -301,8 +328,7 @@ class EtcdJobBoard(base.JobBoard):
try:
job_data = jsonutils.loads(data)
except jsonutils.json.JSONDecodeError:
msg = ("Incorrectly formatted job data found at "
f"key: {key}")
msg = f"Incorrectly formatted job data found at key: {key}"
LOG.warning(msg, exc_info=True)
LOG.info("Deleting invalid job data at key: %s", key)
self._client.delete(key)
@@ -311,16 +337,18 @@ class EtcdJobBoard(base.JobBoard):
with self._job_cond:
if key not in self._job_cache:
job_priority = base.JobPriority.convert(job_data["priority"])
new_job = EtcdJob(self,
job_data["name"],
self._client,
key,
uuid=job_data["uuid"],
details=job_data.get("details", {}),
backend=self._persistence,
book_data=job_data.get("book"),
priority=job_priority,
sequence=job_data["sequence"])
new_job = EtcdJob(
self,
job_data["name"],
self._client,
key,
uuid=job_data["uuid"],
details=job_data.get("details", {}),
backend=self._persistence,
book_data=job_data.get("book"),
priority=job_priority,
sequence=job_data["sequence"],
)
self._job_cache[key] = new_job
self._job_cond.notify_all()
@@ -335,15 +363,18 @@ class EtcdJobBoard(base.JobBoard):
self._remove_job_from_cache(job.key)
self._client.delete_prefix(job.key)
except Exception:
LOG.exception(f"Failed to delete prefix {job.key}")
LOG.exception("Failed to delete prefix %s", job.key)
def iterjobs(self, only_unclaimed=False, ensure_fresh=False):
"""Returns an iterator of jobs that are currently on this board."""
return base.JobBoardIterator(
self, LOG, only_unclaimed=only_unclaimed,
self,
LOG,
only_unclaimed=only_unclaimed,
ensure_fresh=ensure_fresh,
board_fetch_func=self._fetch_jobs,
board_removal_func=self._board_removal_func)
board_removal_func=self._board_removal_func,
)
def wait(self, timeout=None):
"""Waits a given amount of time for **any** jobs to be posted."""
@@ -354,9 +385,10 @@ class EtcdJobBoard(base.JobBoard):
while True:
if not self._job_cache:
if watch.expired():
raise exc.NotFound("Expired waiting for jobs to"
" arrive; waited %s seconds"
% watch.elapsed())
raise exc.NotFound(
"Expired waiting for jobs to"
" arrive; waited %s seconds" % watch.elapsed()
)
# This is done since the given timeout can not be provided
# to the condition variable, since we can not ensure that
# when we acquire the condition that there will actually
@@ -367,10 +399,14 @@ class EtcdJobBoard(base.JobBoard):
curr_jobs = self._fetch_jobs()
fetch_func = lambda ensure_fresh: curr_jobs
removal_func = lambda a_job: self._remove_job_from_cache(
a_job.key)
a_job.key
)
return base.JobBoardIterator(
self, LOG, board_fetch_func=fetch_func,
board_removal_func=removal_func)
self,
LOG,
board_fetch_func=fetch_func,
board_removal_func=removal_func,
)
@property
def job_count(self):
@@ -395,11 +431,11 @@ class EtcdJobBoard(base.JobBoard):
key = job.key + self.DATA_POSTFIX
return self.get_one(key)
def get_owner_and_data(self, job: EtcdJob) -> tuple[
str | None, bytes | None]:
def get_owner_and_data(
self, job: EtcdJob
) -> tuple[str | None, bytes | None]:
if self._client is None:
raise exc.JobFailure("Cannot retrieve information, "
"not connected")
raise exc.JobFailure("Cannot retrieve information, not connected")
job_data = None
job_owner = None
@@ -426,15 +462,20 @@ class EtcdJobBoard(base.JobBoard):
return self.get_one(key)
def post(self, name, book=None, details=None,
priority=base.JobPriority.NORMAL) -> EtcdJob:
def post(
self, name, book=None, details=None, priority=base.JobPriority.NORMAL
) -> EtcdJob:
"""Atomically creates and posts a job to the jobboard."""
job_priority = base.JobPriority.convert(priority)
job_uuid = uuidutils.generate_uuid()
job_posting = base.format_posting(job_uuid, name,
created_on=timeutils.utcnow(),
book=book, details=details,
priority=job_priority)
job_posting = base.format_posting(
job_uuid,
name,
created_on=timeutils.utcnow(),
book=book,
details=details,
priority=job_priority,
)
seq = self.incr(self._create_path(self._root_path, self.SEQUENCE_KEY))
key = self._create_path(self._root_path, f"{self.JOB_PREFIX}{seq}")
@@ -444,14 +485,19 @@ class EtcdJobBoard(base.JobBoard):
data_key = key + self.DATA_POSTFIX
self._client.create(data_key, raw_job_posting)
job = EtcdJob(self, name, self._client, key,
uuid=job_uuid,
details=details,
backend=self._persistence,
book=book,
book_data=job_posting.get('book'),
priority=job_priority,
sequence=seq)
job = EtcdJob(
self,
name,
self._client,
key,
uuid=job_uuid,
details=details,
backend=self._persistence,
book=book,
book_data=job_posting.get('book'),
priority=job_priority,
sequence=seq,
)
with self._job_cond:
self._job_cache[key] = job
self._job_cond.notify_all()
@@ -511,8 +557,9 @@ class EtcdJobBoard(base.JobBoard):
if data is None or owner is None:
raise exc.NotFound(f"Cannot find job {job.uuid}")
if owner != who:
raise exc.JobFailure(f"Cannot consume a job {job.uuid}"
f" which is not owned by {who}")
raise exc.JobFailure(
f"Cannot consume a job {job.uuid} which is not owned by {who}"
)
self._client.delete_prefix(job.key + ".")
self._remove_job_from_cache(job.key)
@@ -524,8 +571,9 @@ class EtcdJobBoard(base.JobBoard):
if data is None or owner is None:
raise exc.NotFound(f"Cannot find job {job.uuid}")
if owner != who:
raise exc.JobFailure(f"Cannot abandon a job {job.uuid}"
f" which is not owned by {who}")
raise exc.JobFailure(
f"Cannot abandon a job {job.uuid} which is not owned by {who}"
)
owner_key = job.key + self.LOCK_POSTFIX
self._client.delete(owner_key)
@@ -537,8 +585,9 @@ class EtcdJobBoard(base.JobBoard):
if data is None or owner is None:
raise exc.NotFound(f"Cannot find job {job.uuid}")
if owner != who:
raise exc.JobFailure(f"Cannot trash a job {job.uuid} "
f"which is not owned by {who}")
raise exc.JobFailure(
f"Cannot trash a job {job.uuid} which is not owned by {who}"
)
trash_key = job.key.replace(self.ROOT_PATH, self.TRASH_PATH)
self._client.create(trash_key, data)
@@ -570,11 +619,13 @@ class EtcdJobBoard(base.JobBoard):
watch_url = self._create_path(self._root_path, self.JOB_PREFIX)
self._thread_cancel = threading.Event()
try:
(self._watcher,
self._watcher_cancel) = self._client.watch_prefix(watch_url)
(self._watcher, self._watcher_cancel) = (
self._client.watch_prefix(watch_url)
)
except etcd3gw.exceptions.ConnectionFailedError:
exc.raise_with_cause(exc.JobFailure,
"Failed to connect to Etcd")
exc.raise_with_cause(
exc.JobFailure, "Failed to connect to Etcd"
)
self._watcher_thd = threading.Thread(target=self._watcher_thread)
self._watcher_thd.start()

530
taskflow/jobs/backends/impl_redis.py
View File

@@ -48,28 +48,43 @@ def _translate_failures():
except redis_exceptions.ConnectionError:
exc.raise_with_cause(exc.JobFailure, "Failed to connect to redis")
except redis_exceptions.TimeoutError:
exc.raise_with_cause(exc.JobFailure,
"Failed to communicate with redis, connection"
" timed out")
exc.raise_with_cause(
exc.JobFailure,
"Failed to communicate with redis, connection timed out",
)
except redis_exceptions.RedisError:
exc.raise_with_cause(exc.JobFailure,
"Failed to communicate with redis,"
" internal error")
exc.raise_with_cause(
exc.JobFailure, "Failed to communicate with redis, internal error"
)
@functools.total_ordering
class RedisJob(base.Job):
"""A redis job."""
def __init__(self, board, name, sequence, key,
uuid=None, details=None,
created_on=None, backend=None,
book=None, book_data=None,
priority=base.JobPriority.NORMAL):
super().__init__(board, name,
uuid=uuid, details=details,
backend=backend,
book=book, book_data=book_data)
def __init__(
self,
board,
name,
sequence,
key,
uuid=None,
details=None,
created_on=None,
backend=None,
book=None,
book_data=None,
priority=base.JobPriority.NORMAL,
):
super().__init__(
board,
name,
uuid=uuid,
details=details,
backend=backend,
book=book,
book_data=book_data,
)
self._created_on = created_on
self._client = board._client
self._redis_version = board._redis_version
@@ -113,8 +128,11 @@ class RedisJob(base.Job):
:attr:`.owner_key` expired at/before time of inquiry?).
"""
with _translate_failures():
return ru.get_expiry(self._client, self._owner_key,
prior_version=self._redis_version)
return ru.get_expiry(
self._client,
self._owner_key,
prior_version=self._redis_version,
)
def extend_expiry(self, expiry):
"""Extends the owner key (aka the claim) expiry for this job.
@@ -128,8 +146,12 @@ class RedisJob(base.Job):
otherwise ``False``.
"""
with _translate_failures():
return ru.apply_expiry(self._client, self._owner_key, expiry,
prior_version=self._redis_version)
return ru.apply_expiry(
self._client,
self._owner_key,
expiry,
prior_version=self._redis_version,
)
def __lt__(self, other):
if not isinstance(other, RedisJob):
@@ -139,7 +161,8 @@ class RedisJob(base.Job):
return self.sequence < other.sequence
else:
ordered = base.JobPriority.reorder(
(self.priority, self), (other.priority, other))
(self.priority, self), (other.priority, other)
)
if ordered[0] is self:
return False
return True
@@ -150,8 +173,11 @@ class RedisJob(base.Job):
def __eq__(self, other):
if not isinstance(other, RedisJob):
return NotImplemented
return ((self.board.listings_key, self.priority, self.sequence) ==
(other.board.listings_key, other.priority, other.sequence))
return (self.board.listings_key, self.priority, self.sequence) == (
other.board.listings_key,
other.priority,
other.sequence,
)
def __ne__(self, other):
return not self.__eq__(other)
@@ -170,7 +196,8 @@ class RedisJob(base.Job):
last_modified = None
if raw_last_modified:
last_modified = self._board._loads(
raw_last_modified, root_types=(datetime.datetime,))
raw_last_modified, root_types=(datetime.datetime,)
)
# NOTE(harlowja): just incase this is somehow busted (due to time
# sync issues/other), give back the most recent one (since redis
# does not maintain clock information; we could have this happen
@@ -199,9 +226,13 @@ class RedisJob(base.Job):
# This should **not** be possible due to lua code ordering
# but let's log an INFO statement if it does happen (so
# that it can be investigated)...
LOG.info("Unexpected owner key found at '%s' when job"
" key '%s[%s]' was not found", owner_key,
listings_key, listings_sub_key)
LOG.info(
"Unexpected owner key found at '%s' when job"
" key '%s[%s]' was not found",
owner_key,
listings_key,
listings_sub_key,
)
return states.COMPLETE
else:
if owner_exists:
@@ -210,9 +241,9 @@ class RedisJob(base.Job):
return states.UNCLAIMED
with _translate_failures():
return self._client.transaction(_do_fetch,
listings_key, owner_key,
value_from_callable=True)
return self._client.transaction(
_do_fetch, listings_key, owner_key, value_from_callable=True
)
class RedisJobBoard(base.JobBoard):
@@ -255,37 +286,33 @@ class RedisJobBoard(base.JobBoard):
.. _hash: https://redis.io/topics/data-types#hashes
"""
CLIENT_CONF_TRANSFERS = tuple([
# Host config...
('host', str),
('port', int),
# See: http://redis.io/commands/auth
('username', str),
('password', str),
# Data encoding/decoding + error handling
('encoding', str),
('encoding_errors', str),
# Connection settings.
('socket_timeout', float),
('socket_connect_timeout', float),
# This one negates the usage of host, port, socket connection
# settings as it doesn't use the same kind of underlying socket...
('unix_socket_path', str),
# Do u want ssl???
('ssl', strutils.bool_from_string),
('ssl_keyfile', str),
('ssl_certfile', str),
('ssl_cert_reqs', str),
('ssl_ca_certs', str),
# See: http://www.rediscookbook.org/multiple_databases.html
('db', int),
])
CLIENT_CONF_TRANSFERS = tuple(
[
# Host config...
('host', str),
('port', int),
# See: http://redis.io/commands/auth
('username', str),
('password', str),
# Data encoding/decoding + error handling
('encoding', str),
('encoding_errors', str),
# Connection settings.
('socket_timeout', float),
('socket_connect_timeout', float),
# This one negates the usage of host, port, socket connection
# settings as it doesn't use the same kind of underlying socket...
('unix_socket_path', str),
# Do u want ssl???
('ssl', strutils.bool_from_string),
('ssl_keyfile', str),
('ssl_certfile', str),
('ssl_cert_reqs', str),
('ssl_ca_certs', str),
# See: http://www.rediscookbook.org/multiple_databases.html
('db', int),
]
)
"""
Keys (and value type converters) that we allow to proxy from the jobboard
configuration into the redis client (used to configure the redis client
@@ -566,8 +593,9 @@ return cmsgpack.pack(result)
@classmethod
def _filter_ssl_options(cls, opts):
if not opts.get('ssl', False):
return {k: v for (k, v) in opts.items()
if not k.startswith('ssl_')}
return {
k: v for (k, v) in opts.items() if not k.startswith('ssl_')
}
return opts
@classmethod
@@ -587,15 +615,14 @@ return cmsgpack.pack(result)
sentinel_kwargs = conf.get('sentinel_kwargs')
if sentinel_kwargs is not None:
sentinel_kwargs = cls._filter_ssl_options(sentinel_kwargs)
s = sentinel.Sentinel(sentinels,
sentinel_kwargs=sentinel_kwargs,
**client_conf)
s = sentinel.Sentinel(
sentinels, sentinel_kwargs=sentinel_kwargs, **client_conf
)
return s.master_for(conf['sentinel'])
else:
return ru.RedisClient(**client_conf)
def __init__(self, name, conf,
client=None, persistence=None):
def __init__(self, name, conf, client=None, persistence=None):
super().__init__(name, conf)
self._closed = True
if client is not None:
@@ -682,25 +709,29 @@ return cmsgpack.pack(result)
# op occurs).
self._client.ping()
is_new_enough, redis_version = ru.is_server_new_enough(
self._client, self.MIN_REDIS_VERSION)
self._client, self.MIN_REDIS_VERSION
)
if not is_new_enough:
wanted_version = ".".join([str(p)
for p in self.MIN_REDIS_VERSION])
wanted_version = ".".join(
[str(p) for p in self.MIN_REDIS_VERSION]
)
if redis_version:
raise exc.JobFailure("Redis version %s or greater is"
" required (version %s is to"
" old)" % (wanted_version,
redis_version))
raise exc.JobFailure(
"Redis version %s or greater is"
" required (version %s is to"
" old)" % (wanted_version, redis_version)
)
else:
raise exc.JobFailure("Redis version %s or greater is"
" required" % (wanted_version))
raise exc.JobFailure(
"Redis version %s or greater is"
" required" % (wanted_version)
)
else:
self._redis_version = redis_version
script_params = {
# Status field values.
'ok': self.SCRIPT_STATUS_OK,
'error': self.SCRIPT_STATUS_ERROR,
# Known error reasons (when status field is error).
'not_expected_owner': self.SCRIPT_NOT_EXPECTED_OWNER,
'unknown_owner': self.SCRIPT_UNKNOWN_OWNER,
@@ -729,18 +760,20 @@ return cmsgpack.pack(result)
try:
return msgpackutils.dumps(obj)
except Exception:
exc.raise_with_cause(exc.JobFailure,
"Failed to serialize object to"
" msgpack blob")
exc.raise_with_cause(
exc.JobFailure, "Failed to serialize object to msgpack blob"
)
@staticmethod
def _loads(blob, root_types=(dict,)):
try:
return misc.decode_msgpack(blob, root_types=root_types)
except ValueError:
exc.raise_with_cause(exc.JobFailure,
"Failed to deserialize object from"
" msgpack blob (of length %s)" % len(blob))
exc.raise_with_cause(
exc.JobFailure,
"Failed to deserialize object from"
" msgpack blob (of length %s)" % len(blob),
)
_decode_owner = staticmethod(misc.binary_decode)
@@ -752,42 +785,66 @@ return cmsgpack.pack(result)
raw_owner = self._client.get(owner_key)
return self._decode_owner(raw_owner)
def post(self, name, book=None, details=None,
priority=base.JobPriority.NORMAL):
def post(
self, name, book=None, details=None, priority=base.JobPriority.NORMAL
):
job_uuid = uuidutils.generate_uuid()
job_priority = base.JobPriority.convert(priority)
posting = base.format_posting(job_uuid, name,
created_on=timeutils.utcnow(),
book=book, details=details,
priority=job_priority)
posting = base.format_posting(
job_uuid,
name,
created_on=timeutils.utcnow(),
book=book,
details=details,
priority=job_priority,
)
with _translate_failures():
sequence = self._client.incr(self.sequence_key)
posting.update({
'sequence': sequence,
})
posting.update(
{
'sequence': sequence,
}
)
with _translate_failures():
raw_posting = self._dumps(posting)
raw_job_uuid = job_uuid.encode('latin-1')
was_posted = bool(self._client.hsetnx(self.listings_key,
raw_job_uuid, raw_posting))
was_posted = bool(
self._client.hsetnx(
self.listings_key, raw_job_uuid, raw_posting
)
)
if not was_posted:
raise exc.JobFailure("New job located at '%s[%s]' could not"
" be posted" % (self.listings_key,
raw_job_uuid))
raise exc.JobFailure(
"New job located at '%s[%s]' could not"
" be posted" % (self.listings_key, raw_job_uuid)
)
else:
return RedisJob(self, name, sequence, raw_job_uuid,
uuid=job_uuid, details=details,
created_on=posting['created_on'],
book=book, book_data=posting.get('book'),
backend=self._persistence,
priority=job_priority)
return RedisJob(
self,
name,
sequence,
raw_job_uuid,
uuid=job_uuid,
details=details,
created_on=posting['created_on'],
book=book,
book_data=posting.get('book'),
backend=self._persistence,
priority=job_priority,
)
def wait(self, timeout=None, initial_delay=0.005,
max_delay=1.0, sleep_func=time.sleep):
def wait(
self,
timeout=None,
initial_delay=0.005,
max_delay=1.0,
sleep_func=time.sleep,
):
if initial_delay > max_delay:
raise ValueError("Initial delay %s must be less than or equal"
" to the provided max delay %s"
% (initial_delay, max_delay))
raise ValueError(
"Initial delay %s must be less than or equal"
" to the provided max delay %s" % (initial_delay, max_delay)
)
# This does a spin-loop that backs off by doubling the delay
# up to the provided max-delay. In the future we could try having
# a secondary client connected into redis pubsub and use that
@@ -801,12 +858,15 @@ return cmsgpack.pack(result)
curr_jobs = self._fetch_jobs()
if curr_jobs:
return base.JobBoardIterator(
self, LOG,
board_fetch_func=lambda ensure_fresh: curr_jobs)
self,
LOG,
board_fetch_func=lambda ensure_fresh: curr_jobs,
)
if w.expired():
raise exc.NotFound("Expired waiting for jobs to"
" arrive; waited %s seconds"
% w.elapsed())
raise exc.NotFound(
"Expired waiting for jobs to"
" arrive; waited %s seconds" % w.elapsed()
)
else:
remaining = w.leftover(return_none=True)
if remaining is not None:
@@ -834,27 +894,43 @@ return cmsgpack.pack(result)
job_details = job_data.get('details', {})
except (ValueError, TypeError, KeyError, exc.JobFailure):
with excutils.save_and_reraise_exception():
LOG.warning("Incorrectly formatted job data found at"
" key: %s[%s]", self.listings_key,
raw_job_key, exc_info=True)
LOG.info("Deleting invalid job data at key: %s[%s]",
self.listings_key, raw_job_key)
LOG.warning(
"Incorrectly formatted job data found at key: %s[%s]",
self.listings_key,
raw_job_key,
exc_info=True,
)
LOG.info(
"Deleting invalid job data at key: %s[%s]",
self.listings_key,
raw_job_key,
)
self._client.hdel(self.listings_key, raw_job_key)
else:
postings.append(RedisJob(self, job_name, job_sequence_id,
raw_job_key, uuid=job_uuid,
details=job_details,
created_on=job_created_on,
book_data=job_data.get('book'),
backend=self._persistence,
priority=job_priority))
postings.append(
RedisJob(
self,
job_name,
job_sequence_id,
raw_job_key,
uuid=job_uuid,
details=job_details,
created_on=job_created_on,
book_data=job_data.get('book'),
backend=self._persistence,
priority=job_priority,
)
)
return sorted(postings, reverse=True)
def iterjobs(self, only_unclaimed=False, ensure_fresh=False):
return base.JobBoardIterator(
self, LOG, only_unclaimed=only_unclaimed,
self,
LOG,
only_unclaimed=only_unclaimed,
ensure_fresh=ensure_fresh,
board_fetch_func=lambda ensure_fresh: self._fetch_jobs())
board_fetch_func=lambda ensure_fresh: self._fetch_jobs(),
)
def register_entity(self, entity):
# Will implement a redis jobboard conductor register later
@@ -865,36 +941,43 @@ return cmsgpack.pack(result)
script = self._get_script('consume')
with _translate_failures():
raw_who = self._encode_owner(who)
raw_result = script(keys=[job.owner_key, self.listings_key,
job.last_modified_key],
args=[raw_who, job.key])
raw_result = script(
keys=[job.owner_key, self.listings_key, job.last_modified_key],
args=[raw_who, job.key],
)
result = self._loads(raw_result)
status = result['status']
if status != self.SCRIPT_STATUS_OK:
reason = result.get('reason')
if reason == self.SCRIPT_UNKNOWN_JOB:
raise exc.NotFound("Job %s not found to be"
" consumed" % (job.uuid))
raise exc.NotFound(
"Job %s not found to be consumed" % (job.uuid)
)
elif reason == self.SCRIPT_UNKNOWN_OWNER:
raise exc.NotFound("Can not consume job %s"
" which we can not determine"
" the owner of" % (job.uuid))
raise exc.NotFound(
"Can not consume job %s"
" which we can not determine"
" the owner of" % (job.uuid)
)
elif reason == self.SCRIPT_NOT_EXPECTED_OWNER:
raw_owner = result.get('owner')
if raw_owner:
owner = self._decode_owner(raw_owner)
raise exc.JobFailure("Can not consume job %s"
" which is not owned by %s (it is"
" actively owned by %s)"
% (job.uuid, who, owner))
raise exc.JobFailure(
"Can not consume job %s"
" which is not owned by %s (it is"
" actively owned by %s)" % (job.uuid, who, owner)
)
else:
raise exc.JobFailure("Can not consume job %s"
" which is not owned by %s"
% (job.uuid, who))
raise exc.JobFailure(
"Can not consume job %s"
" which is not owned by %s" % (job.uuid, who)
)
else:
raise exc.JobFailure("Failure to consume job %s,"
" unknown internal error (reason=%s)"
% (job.uuid, reason))
raise exc.JobFailure(
"Failure to consume job %s,"
" unknown internal error (reason=%s)" % (job.uuid, reason)
)
@base.check_who
def claim(self, job, who, expiry=None):
@@ -906,122 +989,151 @@ return cmsgpack.pack(result)
else:
ms_expiry = int(expiry * 1000.0)
if ms_expiry <= 0:
raise ValueError("Provided expiry (when converted to"
" milliseconds) must be greater"
" than zero instead of %s" % (expiry))
raise ValueError(
"Provided expiry (when converted to"
" milliseconds) must be greater"
" than zero instead of %s" % (expiry)
)
script = self._get_script('claim')
with _translate_failures():
raw_who = self._encode_owner(who)
raw_result = script(keys=[job.owner_key, self.listings_key,
job.last_modified_key],
args=[raw_who, job.key,
# NOTE(harlowja): we need to send this
# in as a blob (even if it's not
# set/used), since the format can not
# currently be created in lua...
self._dumps(timeutils.utcnow()),
ms_expiry])
raw_result = script(
keys=[job.owner_key, self.listings_key, job.last_modified_key],
args=[
raw_who,
job.key,
# NOTE(harlowja): we need to send this
# in as a blob (even if it's not
# set/used), since the format can not
# currently be created in lua...
self._dumps(timeutils.utcnow()),
ms_expiry,
],
)
result = self._loads(raw_result)
status = result['status']
if status != self.SCRIPT_STATUS_OK:
reason = result.get('reason')
if reason == self.SCRIPT_UNKNOWN_JOB:
raise exc.NotFound("Job %s not found to be"
" claimed" % (job.uuid))
raise exc.NotFound(
"Job %s not found to be claimed" % (job.uuid)
)
elif reason == self.SCRIPT_ALREADY_CLAIMED:
raw_owner = result.get('owner')
if raw_owner:
owner = self._decode_owner(raw_owner)
raise exc.UnclaimableJob("Job %s already"
" claimed by %s"
% (job.uuid, owner))
raise exc.UnclaimableJob(
"Job %s already claimed by %s" % (job.uuid, owner)
)
else:
raise exc.UnclaimableJob("Job %s already"
" claimed" % (job.uuid))
raise exc.UnclaimableJob(
"Job %s already claimed" % (job.uuid)
)
else:
raise exc.JobFailure("Failure to claim job %s,"
" unknown internal error (reason=%s)"
% (job.uuid, reason))
raise exc.JobFailure(
"Failure to claim job %s,"
" unknown internal error (reason=%s)" % (job.uuid, reason)
)
@base.check_who
def abandon(self, job, who):
script = self._get_script('abandon')
with _translate_failures():
raw_who = self._encode_owner(who)
raw_result = script(keys=[job.owner_key, self.listings_key,
job.last_modified_key],
args=[raw_who, job.key,
self._dumps(timeutils.utcnow())])
raw_result = script(
keys=[job.owner_key, self.listings_key, job.last_modified_key],
args=[raw_who, job.key, self._dumps(timeutils.utcnow())],
)
result = self._loads(raw_result)
status = result.get('status')
if status != self.SCRIPT_STATUS_OK:
reason = result.get('reason')
if reason == self.SCRIPT_UNKNOWN_JOB:
raise exc.NotFound("Job %s not found to be"
" abandoned" % (job.uuid))
raise exc.NotFound(
"Job %s not found to be abandoned" % (job.uuid)
)
elif reason == self.SCRIPT_UNKNOWN_OWNER:
raise exc.NotFound("Can not abandon job %s"
" which we can not determine"
" the owner of" % (job.uuid))
raise exc.NotFound(
"Can not abandon job %s"
" which we can not determine"
" the owner of" % (job.uuid)
)
elif reason == self.SCRIPT_NOT_EXPECTED_OWNER:
raw_owner = result.get('owner')
if raw_owner:
owner = self._decode_owner(raw_owner)
raise exc.JobFailure("Can not abandon job %s"
" which is not owned by %s (it is"
" actively owned by %s)"
% (job.uuid, who, owner))
raise exc.JobFailure(
"Can not abandon job %s"
" which is not owned by %s (it is"
" actively owned by %s)" % (job.uuid, who, owner)
)
else:
raise exc.JobFailure("Can not abandon job %s"
" which is not owned by %s"
% (job.uuid, who))
raise exc.JobFailure(
"Can not abandon job %s"
" which is not owned by %s" % (job.uuid, who)
)
else:
raise exc.JobFailure("Failure to abandon job %s,"
" unknown internal"
" error (status=%s, reason=%s)"
% (job.uuid, status, reason))
raise exc.JobFailure(
"Failure to abandon job %s,"
" unknown internal"
" error (status=%s, reason=%s)"
% (job.uuid, status, reason)
)
def _get_script(self, name):
try:
return self._scripts[name]
except KeyError:
exc.raise_with_cause(exc.NotFound,
"Can not access %s script (has this"
" board been connected?)" % name)
exc.raise_with_cause(
exc.NotFound,
"Can not access %s script (has this"
" board been connected?)" % name,
)
@base.check_who
def trash(self, job, who):
script = self._get_script('trash')
with _translate_failures():
raw_who = self._encode_owner(who)
raw_result = script(keys=[job.owner_key, self.listings_key,
job.last_modified_key, self.trash_key],
args=[raw_who, job.key,
self._dumps(timeutils.utcnow())])
raw_result = script(
keys=[
job.owner_key,
self.listings_key,
job.last_modified_key,
self.trash_key,
],
args=[raw_who, job.key, self._dumps(timeutils.utcnow())],
)
result = self._loads(raw_result)
status = result['status']
if status != self.SCRIPT_STATUS_OK:
reason = result.get('reason')
if reason == self.SCRIPT_UNKNOWN_JOB:
raise exc.NotFound("Job %s not found to be"
" trashed" % (job.uuid))
raise exc.NotFound(
"Job %s not found to be trashed" % (job.uuid)
)
elif reason == self.SCRIPT_UNKNOWN_OWNER:
raise exc.NotFound("Can not trash job %s"
" which we can not determine"
" the owner of" % (job.uuid))
raise exc.NotFound(
"Can not trash job %s"
" which we can not determine"
" the owner of" % (job.uuid)
)
elif reason == self.SCRIPT_NOT_EXPECTED_OWNER:
raw_owner = result.get('owner')
if raw_owner:
owner = self._decode_owner(raw_owner)
raise exc.JobFailure("Can not trash job %s"
" which is not owned by %s (it is"
" actively owned by %s)"
% (job.uuid, who, owner))
raise exc.JobFailure(
"Can not trash job %s"
" which is not owned by %s (it is"
" actively owned by %s)" % (job.uuid, who, owner)
)
else:
raise exc.JobFailure("Can not trash job %s"
" which is not owned by %s"
% (job.uuid, who))
raise exc.JobFailure(
"Can not trash job %s"
" which is not owned by %s" % (job.uuid, who)
)
else:
raise exc.JobFailure("Failure to trash job %s,"
" unknown internal error (reason=%s)"
% (job.uuid, reason))
raise exc.JobFailure(
"Failure to trash job %s,"
" unknown internal error (reason=%s)" % (job.uuid, reason)
)

441
taskflow/jobs/backends/impl_zookeeper.py
View File

@@ -45,22 +45,37 @@ LOG = logging.getLogger(__name__)
class ZookeeperJob(base.Job):
"""A zookeeper job."""
def __init__(self, board, name, client, path,
uuid=None, details=None, book=None, book_data=None,
created_on=None, backend=None,
priority=base.JobPriority.NORMAL):
super().__init__(board, name,
uuid=uuid, details=details,
backend=backend,
book=book, book_data=book_data)
def __init__(
self,
board,
name,
client,
path,
uuid=None,
details=None,
book=None,
book_data=None,
created_on=None,
backend=None,
priority=base.JobPriority.NORMAL,
):
super().__init__(
board,
name,
uuid=uuid,
details=details,
backend=backend,
book=book,
book_data=book_data,
)
self._client = client
self._path = k_paths.normpath(path)
self._lock_path = self._path + board.LOCK_POSTFIX
self._created_on = created_on
self._node_not_found = False
basename = k_paths.basename(self._path)
self._root = self._path[0:-len(basename)]
self._sequence = int(basename[len(board.JOB_PREFIX):])
self._root = self._path[0 : -len(basename)]
self._sequence = int(basename[len(board.JOB_PREFIX) :])
self._priority = priority
@property
@@ -99,23 +114,26 @@ class ZookeeperJob(base.Job):
excp.raise_with_cause(
excp.NotFound,
"Can not fetch the %r attribute of job %s (%s),"
" path %s not found" % (attr_name, self.uuid,
self.path, path))
" path %s not found" % (attr_name, self.uuid, self.path, path),
)
except self._client.handler.timeout_exception:
excp.raise_with_cause(
excp.JobFailure,
"Can not fetch the %r attribute of job %s (%s),"
" operation timed out" % (attr_name, self.uuid, self.path))
" operation timed out" % (attr_name, self.uuid, self.path),
)
except k_exceptions.SessionExpiredError:
excp.raise_with_cause(
excp.JobFailure,
"Can not fetch the %r attribute of job %s (%s),"
" session expired" % (attr_name, self.uuid, self.path))
" session expired" % (attr_name, self.uuid, self.path),
)
except (AttributeError, k_exceptions.KazooException):
excp.raise_with_cause(
excp.JobFailure,
"Can not fetch the %r attribute of job %s (%s),"
" internal error" % (attr_name, self.uuid, self.path))
" internal error" % (attr_name, self.uuid, self.path),
)
@property
def last_modified(self):
@@ -123,8 +141,8 @@ class ZookeeperJob(base.Job):
try:
if not self._node_not_found:
modified_on = self._get_node_attr(
self.path, 'mtime',
trans_func=misc.millis_to_datetime)
self.path, 'mtime', trans_func=misc.millis_to_datetime
)
except excp.NotFound:
self._node_not_found = True
return modified_on
@@ -137,8 +155,8 @@ class ZookeeperJob(base.Job):
if self._created_on is None:
try:
self._created_on = self._get_node_attr(
self.path, 'ctime',
trans_func=misc.millis_to_datetime)
self.path, 'ctime', trans_func=misc.millis_to_datetime
)
except excp.NotFound:
self._node_not_found = True
return self._created_on
@@ -155,18 +173,19 @@ class ZookeeperJob(base.Job):
except k_exceptions.SessionExpiredError:
excp.raise_with_cause(
excp.JobFailure,
"Can not fetch the state of %s,"
" session expired" % (self.uuid))
"Can not fetch the state of %s, session expired" % (self.uuid),
)
except self._client.handler.timeout_exception:
excp.raise_with_cause(
excp.JobFailure,
"Can not fetch the state of %s,"
" operation timed out" % (self.uuid))
" operation timed out" % (self.uuid),
)
except k_exceptions.KazooException:
excp.raise_with_cause(
excp.JobFailure,
"Can not fetch the state of %s,"
" internal error" % (self.uuid))
"Can not fetch the state of %s, internal error" % (self.uuid),
)
if not job_data:
# No data this job has been completed (the owner that we might have
# fetched will not be able to be fetched again, since the job node
@@ -185,7 +204,8 @@ class ZookeeperJob(base.Job):
return self.sequence < other.sequence
else:
ordered = base.JobPriority.reorder(
(self.priority, self), (other.priority, other))
(self.priority, self), (other.priority, other)
)
if ordered[0] is self:
return False
return True
@@ -196,8 +216,11 @@ class ZookeeperJob(base.Job):
def __eq__(self, other):
if not isinstance(other, ZookeeperJob):
return NotImplemented
return ((self.root, self.sequence, self.priority) ==
(other.root, other.sequence, other.priority))
return (self.root, self.sequence, self.priority) == (
other.root,
other.sequence,
other.priority,
)
def __ne__(self, other):
return not self.__eq__(other)
@@ -277,8 +300,14 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
or may be recovered (aka, it has not full disconnected).
"""
def __init__(self, name, conf,
client=None, persistence=None, emit_notifications=True):
def __init__(
self,
name,
conf,
client=None,
persistence=None,
emit_notifications=True,
):
super().__init__(name, conf)
if client is not None:
self._client = client
@@ -292,11 +321,12 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
if not k_paths.isabs(path):
raise ValueError("Zookeeper path must be absolute")
self._path = path
self._trash_path = self._path.replace(k_paths.basename(self._path),
self.TRASH_FOLDER)
self._trash_path = self._path.replace(
k_paths.basename(self._path), self.TRASH_FOLDER
)
self._entity_path = self._path.replace(
k_paths.basename(self._path),
self.ENTITY_FOLDER)
k_paths.basename(self._path), self.ENTITY_FOLDER
)
# The backend to load the full logbooks from, since what is sent over
# the data connection is only the logbook uuid and name, and not the
# full logbook.
@@ -378,23 +408,30 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
maybe_children = self._client.get_children(self.path)
self._on_job_posting(maybe_children, delayed=False)
except self._client.handler.timeout_exception:
excp.raise_with_cause(excp.JobFailure,
"Refreshing failure, operation timed out")
excp.raise_with_cause(
excp.JobFailure, "Refreshing failure, operation timed out"
)
except k_exceptions.SessionExpiredError:
excp.raise_with_cause(excp.JobFailure,
"Refreshing failure, session expired")
excp.raise_with_cause(
excp.JobFailure, "Refreshing failure, session expired"
)
except k_exceptions.NoNodeError:
pass
except k_exceptions.KazooException:
excp.raise_with_cause(excp.JobFailure,
"Refreshing failure, internal error")
excp.raise_with_cause(
excp.JobFailure, "Refreshing failure, internal error"
)
def iterjobs(self, only_unclaimed=False, ensure_fresh=False):
board_removal_func = lambda job: self._remove_job(job.path)
return base.JobBoardIterator(
self, LOG, only_unclaimed=only_unclaimed,
ensure_fresh=ensure_fresh, board_fetch_func=self._fetch_jobs,
board_removal_func=board_removal_func)
self,
LOG,
only_unclaimed=only_unclaimed,
ensure_fresh=ensure_fresh,
board_fetch_func=self._fetch_jobs,
board_removal_func=board_removal_func,
)
def _remove_job(self, path):
if path not in self._known_jobs:
@@ -424,38 +461,56 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
job_name = job_data['name']
except (ValueError, TypeError, KeyError):
with excutils.save_and_reraise_exception(reraise=not quiet):
LOG.warning("Incorrectly formatted job data found at path: %s",
path, exc_info=True)
LOG.warning(
"Incorrectly formatted job data found at path: %s",
path,
exc_info=True,
)
except self._client.handler.timeout_exception:
with excutils.save_and_reraise_exception(reraise=not quiet):
LOG.warning("Operation timed out fetching job data from"
" from path: %s",
path, exc_info=True)
LOG.warning(
"Operation timed out fetching job data from from path: %s",
path,
exc_info=True,
)
except k_exceptions.SessionExpiredError:
with excutils.save_and_reraise_exception(reraise=not quiet):
LOG.warning("Session expired fetching job data from path: %s",
path, exc_info=True)
LOG.warning(
"Session expired fetching job data from path: %s",
path,
exc_info=True,
)
except k_exceptions.NoNodeError:
LOG.debug("No job node found at path: %s, it must have"
" disappeared or was removed", path)
LOG.debug(
"No job node found at path: %s, it must have"
" disappeared or was removed",
path,
)
except k_exceptions.KazooException:
with excutils.save_and_reraise_exception(reraise=not quiet):
LOG.warning("Internal error fetching job data from path: %s",
path, exc_info=True)
LOG.warning(
"Internal error fetching job data from path: %s",
path,
exc_info=True,
)
else:
with self._job_cond:
# Now we can officially check if someone already placed this
# jobs information into the known job set (if it's already
# existing then just leave it alone).
if path not in self._known_jobs:
job = ZookeeperJob(self, job_name,
self._client, path,
backend=self._persistence,
uuid=job_uuid,
book_data=job_data.get("book"),
details=job_data.get("details", {}),
created_on=job_created_on,
priority=job_priority)
job = ZookeeperJob(
self,
job_name,
self._client,
path,
backend=self._persistence,
uuid=job_uuid,
book_data=job_data.get("book"),
details=job_data.get("details", {}),
created_on=job_created_on,
priority=job_priority,
)
self._known_jobs[path] = job
self._job_cond.notify_all()
if job is not None:
@@ -465,8 +520,9 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
LOG.debug("Got children %s under path %s", children, self.path)
child_paths = []
for c in children:
if (c.endswith(self.LOCK_POSTFIX) or
not c.startswith(self.JOB_PREFIX)):
if c.endswith(self.LOCK_POSTFIX) or not c.startswith(
self.JOB_PREFIX
):
# Skip lock paths or non-job-paths (these are not valid jobs)
continue
child_paths.append(k_paths.join(self.path, c))
@@ -513,29 +569,42 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
else:
self._process_child(path, request, quiet=False)
def post(self, name, book=None, details=None,
priority=base.JobPriority.NORMAL):
def post(
self, name, book=None, details=None, priority=base.JobPriority.NORMAL
):
# NOTE(harlowja): Jobs are not ephemeral, they will persist until they
# are consumed (this may change later, but seems safer to do this until
# further notice).
job_priority = base.JobPriority.convert(priority)
job_uuid = uuidutils.generate_uuid()
job_posting = base.format_posting(job_uuid, name,
book=book, details=details,
priority=job_priority)
job_posting = base.format_posting(
job_uuid, name, book=book, details=details, priority=job_priority
)
raw_job_posting = misc.binary_encode(jsonutils.dumps(job_posting))
with self._wrap(job_uuid, None,
fail_msg_tpl="Posting failure: %s",
ensure_known=False):
job_path = self._client.create(self._job_base,
value=raw_job_posting,
sequence=True,
ephemeral=False)
job = ZookeeperJob(self, name, self._client, job_path,
backend=self._persistence,
book=book, details=details, uuid=job_uuid,
book_data=job_posting.get('book'),
priority=job_priority)
with self._wrap(
job_uuid,
None,
fail_msg_tpl="Posting failure: %s",
ensure_known=False,
):
job_path = self._client.create(
self._job_base,
value=raw_job_posting,
sequence=True,
ephemeral=False,
)
job = ZookeeperJob(
self,
name,
self._client,
job_path,
backend=self._persistence,
book=book,
details=details,
uuid=job_uuid,
book_data=job_posting.get('book'),
priority=job_priority,
)
with self._job_cond:
self._known_jobs[job_path] = job
self._job_cond.notify_all()
@@ -551,19 +620,22 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
owner = None
if owner:
message = "Job {} already claimed by '{}'".format(
job.uuid, owner)
job.uuid, owner
)
else:
message = "Job %s already claimed" % (job.uuid)
excp.raise_with_cause(excp.UnclaimableJob,
message, cause=cause)
excp.raise_with_cause(excp.UnclaimableJob, message, cause=cause)
with self._wrap(job.uuid, job.path,
fail_msg_tpl="Claiming failure: %s"):
with self._wrap(
job.uuid, job.path, fail_msg_tpl="Claiming failure: %s"
):
# NOTE(harlowja): post as json which will allow for future changes
# more easily than a raw string/text.
value = jsonutils.dumps({
'owner': who,
})
value = jsonutils.dumps(
{
'owner': who,
}
)
# Ensure the target job is still existent (at the right version).
job_data, job_stat = self._client.get(job.path)
txn = self._client.transaction()
@@ -571,8 +643,9 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
# removed (somehow...) or updated by someone else to a different
# version...
txn.check(job.path, version=job_stat.version)
txn.create(job.lock_path, value=misc.binary_encode(value),
ephemeral=True)
txn.create(
job.lock_path, value=misc.binary_encode(value), ephemeral=True
)
try:
kazoo_utils.checked_commit(txn)
except k_exceptions.NodeExistsError as e:
@@ -585,24 +658,29 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
excp.raise_with_cause(
excp.NotFound,
"Job %s not found to be claimed" % job.uuid,
cause=e.failures[0])
cause=e.failures[0],
)
if isinstance(e.failures[1], k_exceptions.NodeExistsError):
_unclaimable_try_find_owner(e.failures[1])
else:
excp.raise_with_cause(
excp.UnclaimableJob,
"Job %s claim failed due to transaction"
" not succeeding" % (job.uuid), cause=e)
" not succeeding" % (job.uuid),
cause=e,
)
@contextlib.contextmanager
def _wrap(self, job_uuid, job_path,
fail_msg_tpl="Failure: %s", ensure_known=True):
def _wrap(
self, job_uuid, job_path, fail_msg_tpl="Failure: %s", ensure_known=True
):
if job_path:
fail_msg_tpl += " (%s)" % (job_path)
if ensure_known:
if not job_path:
raise ValueError("Unable to check if %r is a known path"
% (job_path))
raise ValueError(
"Unable to check if %r is a known path" % (job_path)
)
if job_path not in self._known_jobs:
fail_msg_tpl += ", unknown job"
raise excp.NotFound(fail_msg_tpl % (job_uuid))
@@ -622,9 +700,12 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
excp.raise_with_cause(excp.JobFailure, fail_msg_tpl % (job_uuid))
def find_owner(self, job):
with self._wrap(job.uuid, job.path,
fail_msg_tpl="Owner query failure: %s",
ensure_known=False):
with self._wrap(
job.uuid,
job.path,
fail_msg_tpl="Owner query failure: %s",
ensure_known=False,
):
try:
self._client.sync(job.lock_path)
raw_data, _lock_stat = self._client.get(job.lock_path)
@@ -637,8 +718,12 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
def _get_owner_and_data(self, job):
lock_data, lock_stat = self._client.get(job.lock_path)
job_data, job_stat = self._client.get(job.path)
return (misc.decode_json(lock_data), lock_stat,
misc.decode_json(job_data), job_stat)
return (
misc.decode_json(lock_data),
lock_stat,
misc.decode_json(job_data),
job_stat,
)
def register_entity(self, entity):
entity_type = entity.kind
@@ -646,47 +731,58 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
entity_path = k_paths.join(self.entity_path, entity_type)
try:
self._client.ensure_path(entity_path)
self._client.create(k_paths.join(entity_path, entity.name),
value=misc.binary_encode(
jsonutils.dumps(entity.to_dict())),
ephemeral=True)
self._client.create(
k_paths.join(entity_path, entity.name),
value=misc.binary_encode(
jsonutils.dumps(entity.to_dict())
),
ephemeral=True,
)
except k_exceptions.NodeExistsError:
pass
except self._client.handler.timeout_exception:
excp.raise_with_cause(
excp.JobFailure,
"Can not register entity %s under %s, operation"
" timed out" % (entity.name, entity_path))
" timed out" % (entity.name, entity_path),
)
except k_exceptions.SessionExpiredError:
excp.raise_with_cause(
excp.JobFailure,
"Can not register entity %s under %s, session"
" expired" % (entity.name, entity_path))
" expired" % (entity.name, entity_path),
)
except k_exceptions.KazooException:
excp.raise_with_cause(
excp.JobFailure,
"Can not register entity %s under %s, internal"
" error" % (entity.name, entity_path))
" error" % (entity.name, entity_path),
)
else:
raise excp.NotImplementedError(
"Not implemented for other entity type '%s'" % entity_type)
"Not implemented for other entity type '%s'" % entity_type
)
@base.check_who
def consume(self, job, who):
with self._wrap(job.uuid, job.path,
fail_msg_tpl="Consumption failure: %s"):
with self._wrap(
job.uuid, job.path, fail_msg_tpl="Consumption failure: %s"
):
try:
owner_data = self._get_owner_and_data(job)
lock_data, lock_stat, data, data_stat = owner_data
except k_exceptions.NoNodeError:
excp.raise_with_cause(excp.NotFound,
"Can not consume a job %s"
" which we can not determine"
" the owner of" % (job.uuid))
excp.raise_with_cause(
excp.NotFound,
"Can not consume a job %s"
" which we can not determine"
" the owner of" % (job.uuid),
)
if lock_data.get("owner") != who:
raise excp.JobFailure("Can not consume a job %s"
" which is not owned by %s"
% (job.uuid, who))
raise excp.JobFailure(
"Can not consume a job %s"
" which is not owned by %s" % (job.uuid, who)
)
txn = self._client.transaction()
txn.delete(job.lock_path, version=lock_stat.version)
txn.delete(job.path, version=data_stat.version)
@@ -695,40 +791,46 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
@base.check_who
def abandon(self, job, who):
with self._wrap(job.uuid, job.path,
fail_msg_tpl="Abandonment failure: %s"):
with self._wrap(
job.uuid, job.path, fail_msg_tpl="Abandonment failure: %s"
):
try:
owner_data = self._get_owner_and_data(job)
lock_data, lock_stat, data, data_stat = owner_data
except k_exceptions.NoNodeError:
excp.raise_with_cause(excp.NotFound,
"Can not abandon a job %s"
" which we can not determine"
" the owner of" % (job.uuid))
excp.raise_with_cause(
excp.NotFound,
"Can not abandon a job %s"
" which we can not determine"
" the owner of" % (job.uuid),
)
if lock_data.get("owner") != who:
raise excp.JobFailure("Can not abandon a job %s"
" which is not owned by %s"
% (job.uuid, who))
raise excp.JobFailure(
"Can not abandon a job %s"
" which is not owned by %s" % (job.uuid, who)
)
txn = self._client.transaction()
txn.delete(job.lock_path, version=lock_stat.version)
kazoo_utils.checked_commit(txn)
@base.check_who
def trash(self, job, who):
with self._wrap(job.uuid, job.path,
fail_msg_tpl="Trash failure: %s"):
with self._wrap(job.uuid, job.path, fail_msg_tpl="Trash failure: %s"):
try:
owner_data = self._get_owner_and_data(job)
lock_data, lock_stat, data, data_stat = owner_data
except k_exceptions.NoNodeError:
excp.raise_with_cause(excp.NotFound,
"Can not trash a job %s"
" which we can not determine"
" the owner of" % (job.uuid))
excp.raise_with_cause(
excp.NotFound,
"Can not trash a job %s"
" which we can not determine"
" the owner of" % (job.uuid),
)
if lock_data.get("owner") != who:
raise excp.JobFailure("Can not trash a job %s"
" which is not owned by %s"
% (job.uuid, who))
raise excp.JobFailure(
"Can not trash a job %s"
" which is not owned by %s" % (job.uuid, who)
)
trash_path = job.path.replace(self.path, self.trash_path)
value = misc.binary_encode(jsonutils.dumps(data))
txn = self._client.transaction()
@@ -739,12 +841,18 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
def _state_change_listener(self, state):
if self._last_states:
LOG.debug("Kazoo client has changed to"
" state '%s' from prior states '%s'", state,
self._last_states)
LOG.debug(
"Kazoo client has changed to"
" state '%s' from prior states '%s'",
state,
self._last_states,
)
else:
LOG.debug("Kazoo client has changed to state '%s' (from"
" its initial/uninitialized state)", state)
LOG.debug(
"Kazoo client has changed to state '%s' (from"
" its initial/uninitialized state)",
state,
)
self._last_states.appendleft(state)
if state == k_states.KazooState.LOST:
self._connected = False
@@ -769,9 +877,10 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
while True:
if not self._known_jobs:
if watch.expired():
raise excp.NotFound("Expired waiting for jobs to"
" arrive; waited %s seconds"
% watch.elapsed())
raise excp.NotFound(
"Expired waiting for jobs to"
" arrive; waited %s seconds" % watch.elapsed()
)
# This is done since the given timeout can not be provided
# to the condition variable, since we can not ensure that
# when we acquire the condition that there will actually
@@ -783,8 +892,11 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
fetch_func = lambda ensure_fresh: curr_jobs
removal_func = lambda a_job: self._remove_job(a_job.path)
return base.JobBoardIterator(
self, LOG, board_fetch_func=fetch_func,
board_removal_func=removal_func)
self,
LOG,
board_fetch_func=fetch_func,
board_removal_func=removal_func,
)
@property
def connected(self):
@@ -816,21 +928,27 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
try:
self.close()
except k_exceptions.KazooException:
LOG.exception("Failed cleaning-up after post-connection"
" initialization failed")
LOG.exception(
"Failed cleaning-up after post-connection"
" initialization failed"
)
try:
if timeout is not None:
timeout = float(timeout)
self._client.start(timeout=timeout)
self._closing = False
except (self._client.handler.timeout_exception,
k_exceptions.KazooException):
excp.raise_with_cause(excp.JobFailure,
"Failed to connect to zookeeper")
except (
self._client.handler.timeout_exception,
k_exceptions.KazooException,
):
excp.raise_with_cause(
excp.JobFailure, "Failed to connect to zookeeper"
)
try:
if strutils.bool_from_string(
self._conf.get('check_compatible'), default=True):
self._conf.get('check_compatible'), default=True
):
kazoo_utils.check_compatible(self._client, self.MIN_ZK_VERSION)
if self._worker is None and self._emit_notifications:
self._worker = futurist.ThreadPoolExecutor(max_workers=1)
@@ -841,18 +959,23 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
self._client,
self.path,
func=self._on_job_posting,
allow_session_lost=True)
allow_session_lost=True,
)
self._connected = True
except excp.IncompatibleVersion:
with excutils.save_and_reraise_exception():
try_clean()
except (self._client.handler.timeout_exception,
k_exceptions.KazooException):
except (
self._client.handler.timeout_exception,
k_exceptions.KazooException,
):
exc_type, exc, exc_tb = sys.exc_info()
try:
try_clean()
excp.raise_with_cause(excp.JobFailure,
"Failed to do post-connection"
" initialization", cause=exc)
excp.raise_with_cause(
excp.JobFailure,
"Failed to do post-connection initialization",
cause=exc,
)
finally:
del (exc_type, exc, exc_tb)

111
taskflow/jobs/base.py
View File

@@ -59,18 +59,24 @@ class JobPriority(enum.Enum):
try:
return cls(value.upper())
except (ValueError, AttributeError):
valids = [cls.VERY_HIGH, cls.HIGH, cls.NORMAL,
cls.LOW, cls.VERY_LOW]
valids = [
cls.VERY_HIGH,
cls.HIGH,
cls.NORMAL,
cls.LOW,
cls.VERY_LOW,
]
valids = [p.value for p in valids]
raise ValueError("'%s' is not a valid priority, valid"
" priorities are %s" % (value, valids))
raise ValueError(
"'%s' is not a valid priority, valid"
" priorities are %s" % (value, valids)
)
@classmethod
def reorder(cls, *values):
"""Reorders (priority, value) tuples -> priority ordered values."""
if len(values) == 0:
raise ValueError("At least one (priority, value) pair is"
" required")
raise ValueError("At least one (priority, value) pair is required")
elif len(values) == 1:
v1 = values[0]
# Even though this isn't used, we do the conversion because
@@ -81,8 +87,13 @@ class JobPriority(enum.Enum):
return v1[1]
else:
# Order very very much matters in this tuple...
priority_ordering = (cls.VERY_HIGH, cls.HIGH,
cls.NORMAL, cls.LOW, cls.VERY_LOW)
priority_ordering = (
cls.VERY_HIGH,
cls.HIGH,
cls.NORMAL,
cls.LOW,
cls.VERY_LOW,
)
if len(values) == 2:
# It's common to use this in a 2 tuple situation, so
# make it avoid all the needed complexity that is done
@@ -99,7 +110,7 @@ class JobPriority(enum.Enum):
return v2[1], v1[1]
else:
buckets = collections.defaultdict(list)
for (p, v) in values:
for p, v in values:
p = cls.convert(p)
buckets[p].append(v)
values = []
@@ -127,9 +138,16 @@ class Job(metaclass=abc.ABCMeta):
reverting...
"""
def __init__(self, board, name,
uuid=None, details=None, backend=None,
book=None, book_data=None):
def __init__(
self,
board,
name,
uuid=None,
details=None,
backend=None,
book=None,
book_data=None,
):
if uuid:
self._uuid = uuid
else:
@@ -170,9 +188,14 @@ class Job(metaclass=abc.ABCMeta):
def priority(self):
"""The :py:class:`~.JobPriority` of this job."""
def wait(self, timeout=None,
delay=0.01, delay_multiplier=2.0, max_delay=60.0,
sleep_func=time.sleep):
def wait(
self,
timeout=None,
delay=0.01,
delay_multiplier=2.0,
max_delay=60.0,
sleep_func=time.sleep,
):
"""Wait for job to enter completion state.
If the job has not completed in the given timeout, then return false,
@@ -194,8 +217,9 @@ class Job(metaclass=abc.ABCMeta):
w.start()
else:
w = None
delay_gen = iter_utils.generate_delays(delay, max_delay,
multiplier=delay_multiplier)
delay_gen = iter_utils.generate_delays(
delay, max_delay, multiplier=delay_multiplier
)
while True:
if w is not None and w.expired():
return False
@@ -254,11 +278,14 @@ class Job(metaclass=abc.ABCMeta):
"""The non-uniquely identifying name of this job."""
return self._name
@tenacity.retry(retry=tenacity.retry_if_exception_type(
exception_types=excp.StorageFailure),
stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS),
wait=tenacity.wait_fixed(RETRY_WAIT_TIMEOUT),
reraise=True)
@tenacity.retry(
retry=tenacity.retry_if_exception_type(
exception_types=excp.StorageFailure
),
stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS),
wait=tenacity.wait_fixed(RETRY_WAIT_TIMEOUT),
reraise=True,
)
def _load_book(self):
book_uuid = self.book_uuid
if self._backend is not None and book_uuid is not None:
@@ -276,8 +303,8 @@ class Job(metaclass=abc.ABCMeta):
"""Pretty formats the job into something *more* meaningful."""
cls_name = type(self).__name__
return "{}: {} (priority={}, uuid={}, details={})".format(
cls_name, self.name, self.priority,
self.uuid, self.details)
cls_name, self.name, self.priority, self.uuid, self.details
)
class JobBoardIterator:
@@ -296,9 +323,15 @@ class JobBoardIterator:
_UNCLAIMED_JOB_STATES = (states.UNCLAIMED,)
_JOB_STATES = (states.UNCLAIMED, states.COMPLETE, states.CLAIMED)
def __init__(self, board, logger,
board_fetch_func=None, board_removal_func=None,
only_unclaimed=False, ensure_fresh=False):
def __init__(
self,
board,
logger,
board_fetch_func=None,
board_removal_func=None,
only_unclaimed=False,
ensure_fresh=False,
):
self._board = board
self._logger = logger
self._board_removal_func = board_removal_func
@@ -328,8 +361,11 @@ class JobBoardIterator:
if maybe_job.state in allowed_states:
job = maybe_job
except excp.JobFailure:
self._logger.warning("Failed determining the state of"
" job '%s'", maybe_job, exc_info=True)
self._logger.warning(
"Failed determining the state of job '%s'",
maybe_job,
exc_info=True,
)
except excp.NotFound:
# Attempt to clean this off the board now that we found
# it wasn't really there (this **must** gracefully handle
@@ -343,8 +379,8 @@ class JobBoardIterator:
if not self._fetched:
if self._board_fetch_func is not None:
self._jobs.extend(
self._board_fetch_func(
ensure_fresh=self.ensure_fresh))
self._board_fetch_func(ensure_fresh=self.ensure_fresh)
)
self._fetched = True
job = self._next_job()
if job is None:
@@ -562,6 +598,7 @@ class NotifyingJobBoard(JobBoard):
separate dedicated thread when they occur, so ensure that all callbacks
registered are thread safe (and block for as little time as possible).
"""
def __init__(self, name, conf):
super().__init__(name, conf)
self.notifier = notifier.Notifier()
@@ -569,6 +606,7 @@ class NotifyingJobBoard(JobBoard):
# Internal helpers for usage by board implementations...
def check_who(meth):
@functools.wraps(meth)
@@ -582,8 +620,15 @@ def check_who(meth):
return wrapper
def format_posting(uuid, name, created_on=None, last_modified=None,
details=None, book=None, priority=JobPriority.NORMAL):
def format_posting(
uuid,
name,
created_on=None,
last_modified=None,
details=None,
book=None,
priority=JobPriority.NORMAL,
):
posting = {
'uuid': uuid,
'name': name,

120
taskflow/listeners/base.py
View File

@@ -24,8 +24,12 @@ from taskflow.types import notifier
LOG = logging.getLogger(__name__)
#: These states will results be usable, other states do not produce results.
FINISH_STATES = (states.FAILURE, states.SUCCESS,
states.REVERTED, states.REVERT_FAILURE)
FINISH_STATES = (
states.FAILURE,
states.SUCCESS,
states.REVERTED,
states.REVERT_FAILURE,
)
#: What is listened for by default...
DEFAULT_LISTEN_FOR = (notifier.Notifier.ANY,)
@@ -53,8 +57,7 @@ def _bulk_deregister(notifier, registered, details_filter=None):
"""Bulk deregisters callbacks associated with many states."""
while registered:
state, cb = registered.pop()
notifier.deregister(state, cb,
details_filter=details_filter)
notifier.deregister(state, cb, details_filter=details_filter)
def _bulk_register(watch_states, notifier, cb, details_filter=None):
@@ -62,15 +65,16 @@ def _bulk_register(watch_states, notifier, cb, details_filter=None):
registered = []
try:
for state in watch_states:
if not notifier.is_registered(state, cb,
details_filter=details_filter):
notifier.register(state, cb,
details_filter=details_filter)
if not notifier.is_registered(
state, cb, details_filter=details_filter
):
notifier.register(state, cb, details_filter=details_filter)
registered.append((state, cb))
except ValueError:
with excutils.save_and_reraise_exception():
_bulk_deregister(notifier, registered,
details_filter=details_filter)
_bulk_deregister(
notifier, registered, details_filter=details_filter
)
else:
return registered
@@ -88,10 +92,13 @@ class Listener:
methods (in this class, they do nothing).
"""
def __init__(self, engine,
task_listen_for=DEFAULT_LISTEN_FOR,
flow_listen_for=DEFAULT_LISTEN_FOR,
retry_listen_for=DEFAULT_LISTEN_FOR):
def __init__(
self,
engine,
task_listen_for=DEFAULT_LISTEN_FOR,
flow_listen_for=DEFAULT_LISTEN_FOR,
retry_listen_for=DEFAULT_LISTEN_FOR,
):
if not task_listen_for:
task_listen_for = []
if not retry_listen_for:
@@ -117,33 +124,44 @@ class Listener:
def deregister(self):
if 'task' in self._registered:
_bulk_deregister(self._engine.atom_notifier,
self._registered['task'],
details_filter=_task_matcher)
_bulk_deregister(
self._engine.atom_notifier,
self._registered['task'],
details_filter=_task_matcher,
)
del self._registered['task']
if 'retry' in self._registered:
_bulk_deregister(self._engine.atom_notifier,
self._registered['retry'],
details_filter=_retry_matcher)
_bulk_deregister(
self._engine.atom_notifier,
self._registered['retry'],
details_filter=_retry_matcher,
)
del self._registered['retry']
if 'flow' in self._registered:
_bulk_deregister(self._engine.notifier,
self._registered['flow'])
_bulk_deregister(self._engine.notifier, self._registered['flow'])
del self._registered['flow']
def register(self):
if 'task' not in self._registered:
self._registered['task'] = _bulk_register(
self._listen_for['task'], self._engine.atom_notifier,
self._task_receiver, details_filter=_task_matcher)
self._listen_for['task'],
self._engine.atom_notifier,
self._task_receiver,
details_filter=_task_matcher,
)
if 'retry' not in self._registered:
self._registered['retry'] = _bulk_register(
self._listen_for['retry'], self._engine.atom_notifier,
self._retry_receiver, details_filter=_retry_matcher)
self._listen_for['retry'],
self._engine.atom_notifier,
self._retry_receiver,
details_filter=_retry_matcher,
)
if 'flow' not in self._registered:
self._registered['flow'] = _bulk_register(
self._listen_for['flow'], self._engine.notifier,
self._flow_receiver)
self._listen_for['flow'],
self._engine.notifier,
self._flow_receiver,
)
def __enter__(self):
self.register()
@@ -154,8 +172,11 @@ class Listener:
self.deregister()
except Exception:
# Don't let deregistering throw exceptions
LOG.warning("Failed deregistering listeners from engine %s",
self._engine, exc_info=True)
LOG.warning(
"Failed deregistering listeners from engine %s",
self._engine,
exc_info=True,
)
class DumpingListener(Listener, metaclass=abc.ABCMeta):
@@ -174,9 +195,14 @@ class DumpingListener(Listener, metaclass=abc.ABCMeta):
"""Dumps the provided *templated* message to some output."""
def _flow_receiver(self, state, details):
self._dump("%s has moved flow '%s' (%s) into state '%s'"
" from state '%s'", self._engine, details['flow_name'],
details['flow_uuid'], state, details['old_state'])
self._dump(
"%s has moved flow '%s' (%s) into state '%s' from state '%s'",
self._engine,
details['flow_name'],
details['flow_uuid'],
state,
details['old_state'],
)
def _task_receiver(self, state, details):
if state in FINISH_STATES:
@@ -187,12 +213,24 @@ class DumpingListener(Listener, metaclass=abc.ABCMeta):
if result.exc_info:
exc_info = tuple(result.exc_info)
was_failure = True
self._dump("%s has moved task '%s' (%s) into state '%s'"
" from state '%s' with result '%s' (failure=%s)",
self._engine, details['task_name'],
details['task_uuid'], state, details['old_state'],
result, was_failure, exc_info=exc_info)
self._dump(
"%s has moved task '%s' (%s) into state '%s'"
" from state '%s' with result '%s' (failure=%s)",
self._engine,
details['task_name'],
details['task_uuid'],
state,
details['old_state'],
result,
was_failure,
exc_info=exc_info,
)
else:
self._dump("%s has moved task '%s' (%s) into state '%s'"
" from state '%s'", self._engine, details['task_name'],
details['task_uuid'], state, details['old_state'])
self._dump(
"%s has moved task '%s' (%s) into state '%s' from state '%s'",
self._engine,
details['task_name'],
details['task_uuid'],
state,
details['old_state'],
)

49
taskflow/listeners/capturing.py
View File

@@ -51,23 +51,31 @@ class CaptureListener(base.Listener):
#: Kind that denotes a 'retry' capture.
RETRY = 'retry'
def __init__(self, engine,
task_listen_for=base.DEFAULT_LISTEN_FOR,
flow_listen_for=base.DEFAULT_LISTEN_FOR,
retry_listen_for=base.DEFAULT_LISTEN_FOR,
# Easily override what you want captured and where it
# should save into and what should be skipped...
capture_flow=True, capture_task=True, capture_retry=True,
# Skip capturing *all* tasks, all retries, all flows...
skip_tasks=None, skip_retries=None, skip_flows=None,
# Provide your own list (or previous list) to accumulate
# into...
values=None):
def __init__(
self,
engine,
task_listen_for=base.DEFAULT_LISTEN_FOR,
flow_listen_for=base.DEFAULT_LISTEN_FOR,
retry_listen_for=base.DEFAULT_LISTEN_FOR,
# Easily override what you want captured and where it
# should save into and what should be skipped...
capture_flow=True,
capture_task=True,
capture_retry=True,
# Skip capturing *all* tasks, all retries, all flows...
skip_tasks=None,
skip_retries=None,
skip_flows=None,
# Provide your own list (or previous list) to accumulate
# into...
values=None,
):
super().__init__(
engine,
task_listen_for=task_listen_for,
flow_listen_for=flow_listen_for,
retry_listen_for=retry_listen_for)
retry_listen_for=retry_listen_for,
)
self._capture_flow = capture_flow
self._capture_task = capture_task
self._capture_retry = capture_retry
@@ -87,17 +95,20 @@ class CaptureListener(base.Listener):
def _task_receiver(self, state, details):
if self._capture_task:
if details['task_name'] not in self._skip_tasks:
self.values.append(self._format_capture(self.TASK,
state, details))
self.values.append(
self._format_capture(self.TASK, state, details)
)
def _retry_receiver(self, state, details):
if self._capture_retry:
if details['retry_name'] not in self._skip_retries:
self.values.append(self._format_capture(self.RETRY,
state, details))
self.values.append(
self._format_capture(self.RETRY, state, details)
)
def _flow_receiver(self, state, details):
if self._capture_flow:
if details['flow_name'] not in self._skip_flows:
self.values.append(self._format_capture(self.FLOW,
state, details))
self.values.append(
self._format_capture(self.FLOW, state, details)
)

31
taskflow/listeners/claims.py
View File

@@ -55,8 +55,9 @@ class CheckingClaimListener(base.Listener):
self._on_job_loss = self._suspend_engine_on_loss
else:
if not callable(on_job_loss):
raise ValueError("Custom 'on_job_loss' handler must be"
" callable")
raise ValueError(
"Custom 'on_job_loss' handler must be callable"
)
self._on_job_loss = on_job_loss
def _suspend_engine_on_loss(self, engine, state, details):
@@ -64,9 +65,14 @@ class CheckingClaimListener(base.Listener):
try:
engine.suspend()
except exceptions.TaskFlowException as e:
LOG.warning("Failed suspending engine '%s', (previously owned by"
" '%s'):%s%s", engine, self._owner, os.linesep,
e.pformat())
LOG.warning(
"Failed suspending engine '%s', (previously owned by"
" '%s'):%s%s",
engine,
self._owner,
os.linesep,
e.pformat(),
)
def _flow_receiver(self, state, details):
self._claim_checker(state, details)
@@ -88,10 +94,15 @@ class CheckingClaimListener(base.Listener):
def _claim_checker(self, state, details):
if not self._has_been_lost():
LOG.debug("Job '%s' is still claimed (actively owned by '%s')",
self._job, self._owner)
LOG.debug(
"Job '%s' is still claimed (actively owned by '%s')",
self._job,
self._owner,
)
else:
LOG.warning("Job '%s' has lost its claim"
" (previously owned by '%s')",
self._job, self._owner)
LOG.warning(
"Job '%s' has lost its claim (previously owned by '%s')",
self._job,
self._owner,
)
self._on_job_loss(self._engine, state, details)

153
taskflow/listeners/logging.py
View File

@@ -38,15 +38,21 @@ class LoggingListener(base.DumpingListener):
#: Default logger to use if one is not provided on construction.
_LOGGER = None
def __init__(self, engine,
task_listen_for=base.DEFAULT_LISTEN_FOR,
flow_listen_for=base.DEFAULT_LISTEN_FOR,
retry_listen_for=base.DEFAULT_LISTEN_FOR,
log=None,
level=logging.DEBUG):
def __init__(
self,
engine,
task_listen_for=base.DEFAULT_LISTEN_FOR,
flow_listen_for=base.DEFAULT_LISTEN_FOR,
retry_listen_for=base.DEFAULT_LISTEN_FOR,
log=None,
level=logging.DEBUG,
):
super().__init__(
engine, task_listen_for=task_listen_for,
flow_listen_for=flow_listen_for, retry_listen_for=retry_listen_for)
engine,
task_listen_for=task_listen_for,
flow_listen_for=flow_listen_for,
retry_listen_for=retry_listen_for,
)
self._logger = misc.pick_first_not_none(log, self._LOGGER, LOG)
self._level = level
@@ -102,18 +108,26 @@ class DynamicLoggingListener(base.Listener):
#: States which are triggered under some type of failure.
_FAILURE_STATES = (states.FAILURE, states.REVERT_FAILURE)
def __init__(self, engine,
task_listen_for=base.DEFAULT_LISTEN_FOR,
flow_listen_for=base.DEFAULT_LISTEN_FOR,
retry_listen_for=base.DEFAULT_LISTEN_FOR,
log=None, failure_level=logging.WARNING,
level=logging.DEBUG, hide_inputs_outputs_of=(),
fail_formatter=None,
mask_inputs_keys=(),
mask_outputs_keys=()):
def __init__(
self,
engine,
task_listen_for=base.DEFAULT_LISTEN_FOR,
flow_listen_for=base.DEFAULT_LISTEN_FOR,
retry_listen_for=base.DEFAULT_LISTEN_FOR,
log=None,
failure_level=logging.WARNING,
level=logging.DEBUG,
hide_inputs_outputs_of=(),
fail_formatter=None,
mask_inputs_keys=(),
mask_outputs_keys=(),
):
super().__init__(
engine, task_listen_for=task_listen_for,
flow_listen_for=flow_listen_for, retry_listen_for=retry_listen_for)
engine,
task_listen_for=task_listen_for,
flow_listen_for=flow_listen_for,
retry_listen_for=retry_listen_for,
)
self._failure_level = failure_level
self._level = level
self._task_log_levels = {
@@ -135,16 +149,22 @@ class DynamicLoggingListener(base.Listener):
self._engine,
hide_inputs_outputs_of=self._hide_inputs_outputs_of,
mask_inputs_keys=self._mask_inputs_keys,
mask_outputs_keys=self._mask_outputs_keys)
mask_outputs_keys=self._mask_outputs_keys,
)
else:
self._fail_formatter = fail_formatter
def _flow_receiver(self, state, details):
"""Gets called on flow state changes."""
level = self._flow_log_levels.get(state, self._level)
self._logger.log(level, "Flow '%s' (%s) transitioned into state '%s'"
" from state '%s'", details['flow_name'],
details['flow_uuid'], state, details.get('old_state'))
self._logger.log(
level,
"Flow '%s' (%s) transitioned into state '%s' from state '%s'",
details['flow_name'],
details['flow_uuid'],
state,
details.get('old_state'),
)
def _task_receiver(self, state, details):
"""Gets called on task state changes."""
@@ -156,41 +176,74 @@ class DynamicLoggingListener(base.Listener):
result = details.get('result')
if isinstance(result, failure.Failure):
exc_info, fail_details = self._fail_formatter.format(
result, _make_matcher(task_name))
result, _make_matcher(task_name)
)
if fail_details:
self._logger.log(self._failure_level,
"Task '%s' (%s) transitioned into state"
" '%s' from state '%s'%s%s",
task_name, task_uuid, state,
details['old_state'], os.linesep,
fail_details, exc_info=exc_info)
self._logger.log(
self._failure_level,
"Task '%s' (%s) transitioned into state"
" '%s' from state '%s'%s%s",
task_name,
task_uuid,
state,
details['old_state'],
os.linesep,
fail_details,
exc_info=exc_info,
)
else:
self._logger.log(self._failure_level,
"Task '%s' (%s) transitioned into state"
" '%s' from state '%s'", task_name,
task_uuid, state, details['old_state'],
exc_info=exc_info)
self._logger.log(
self._failure_level,
"Task '%s' (%s) transitioned into state"
" '%s' from state '%s'",
task_name,
task_uuid,
state,
details['old_state'],
exc_info=exc_info,
)
else:
# Otherwise, depending on the enabled logging level/state we
# will show or hide results that the task may have produced
# during execution.
level = self._task_log_levels.get(state, self._level)
show_result = (self._logger.isEnabledFor(self._level)
or state == states.FAILURE)
if show_result and \
task_name not in self._hide_inputs_outputs_of:
self._logger.log(level, "Task '%s' (%s) transitioned into"
" state '%s' from state '%s' with"
" result '%s'", task_name, task_uuid,
state, details['old_state'], result)
show_result = (
self._logger.isEnabledFor(self._level)
or state == states.FAILURE
)
if (
show_result
and task_name not in self._hide_inputs_outputs_of
):
self._logger.log(
level,
"Task '%s' (%s) transitioned into"
" state '%s' from state '%s' with"
" result '%s'",
task_name,
task_uuid,
state,
details['old_state'],
result,
)
else:
self._logger.log(level, "Task '%s' (%s) transitioned into"
" state '%s' from state '%s'",
task_name, task_uuid, state,
details['old_state'])
self._logger.log(
level,
"Task '%s' (%s) transitioned into"
" state '%s' from state '%s'",
task_name,
task_uuid,
state,
details['old_state'],
)
else:
# Just a intermediary state, carry on!
level = self._task_log_levels.get(state, self._level)
self._logger.log(level, "Task '%s' (%s) transitioned into state"
" '%s' from state '%s'", task_name, task_uuid,
state, details['old_state'])
self._logger.log(
level,
"Task '%s' (%s) transitioned into state '%s' from state '%s'",
task_name,
task_uuid,
state,
details['old_state'],
)

26
taskflow/listeners/printing.py
View File

@@ -20,14 +20,21 @@ from taskflow.listeners import base
class PrintingListener(base.DumpingListener):
"""Writes the task and flow notifications messages to stdout or stderr."""
def __init__(self, engine,
task_listen_for=base.DEFAULT_LISTEN_FOR,
flow_listen_for=base.DEFAULT_LISTEN_FOR,
retry_listen_for=base.DEFAULT_LISTEN_FOR,
stderr=False):
def __init__(
self,
engine,
task_listen_for=base.DEFAULT_LISTEN_FOR,
flow_listen_for=base.DEFAULT_LISTEN_FOR,
retry_listen_for=base.DEFAULT_LISTEN_FOR,
stderr=False,
):
super().__init__(
engine, task_listen_for=task_listen_for,
flow_listen_for=flow_listen_for, retry_listen_for=retry_listen_for)
engine,
task_listen_for=task_listen_for,
flow_listen_for=flow_listen_for,
retry_listen_for=retry_listen_for,
)
if stderr:
self._file = sys.stderr
else:
@@ -37,5 +44,6 @@ class PrintingListener(base.DumpingListener):
print(message % args, file=self._file)
exc_info = kwargs.get('exc_info')
if exc_info is not None:
traceback.print_exception(exc_info[0], exc_info[1], exc_info[2],
file=self._file)
traceback.print_exception(
exc_info[0], exc_info[1], exc_info[2], file=self._file
)

73
taskflow/listeners/timing.py
View File

@@ -25,8 +25,9 @@ from taskflow import states
STARTING_STATES = frozenset((states.RUNNING, states.REVERTING))
FINISHED_STATES = frozenset(base.FINISH_STATES + (states.REVERTED,))
WATCH_STATES = frozenset(itertools.chain(FINISHED_STATES, STARTING_STATES,
[states.PENDING]))
WATCH_STATES = frozenset(
itertools.chain(FINISHED_STATES, STARTING_STATES, [states.PENDING])
)
LOG = logging.getLogger(__name__)
@@ -45,10 +46,11 @@ class DurationListener(base.Listener):
to storage. It saves the duration in seconds as float value
to task metadata with key ``'duration'``.
"""
def __init__(self, engine):
super().__init__(engine,
task_listen_for=WATCH_STATES,
flow_listen_for=WATCH_STATES)
super().__init__(
engine, task_listen_for=WATCH_STATES, flow_listen_for=WATCH_STATES
)
self._timers = {co.TASK: {}, co.FLOW: {}}
def deregister(self):
@@ -58,9 +60,12 @@ class DurationListener(base.Listener):
for item_type, timers in self._timers.items():
leftover_timers = len(timers)
if leftover_timers:
LOG.warning("%s %s(s) did not enter %s states",
leftover_timers,
item_type, FINISHED_STATES)
LOG.warning(
"%s %s(s) did not enter %s states",
leftover_timers,
item_type,
FINISHED_STATES,
)
timers.clear()
def _record_ending(self, timer, item_type, item_name, state):
@@ -76,8 +81,13 @@ class DurationListener(base.Listener):
else:
storage.update_atom_metadata(item_name, meta_update)
except exc.StorageFailure:
LOG.warning("Failure to store duration update %s for %s %s",
meta_update, item_type, item_name, exc_info=True)
LOG.warning(
"Failure to store duration update %s for %s %s",
meta_update,
item_type,
item_name,
exc_info=True,
)
def _task_receiver(self, state, details):
task_name = details['task_name']
@@ -110,10 +120,11 @@ class PrintingDurationListener(DurationListener):
self._printer = printer
def _record_ending(self, timer, item_type, item_name, state):
super()._record_ending(
timer, item_type, item_name, state)
self._printer("It took %s '%s' %0.2f seconds to"
" finish." % (item_type, item_name, timer.elapsed()))
super()._record_ending(timer, item_type, item_name, state)
self._printer(
"It took %s '%s' %0.2f seconds to"
" finish." % (item_type, item_name, timer.elapsed())
)
def _receiver(self, item_type, item_name, state):
super()._receiver(item_type, item_name, state)
@@ -132,13 +143,19 @@ class EventTimeListener(base.Listener):
This information can be later extracted/examined to derive durations...
"""
def __init__(self, engine,
task_listen_for=base.DEFAULT_LISTEN_FOR,
flow_listen_for=base.DEFAULT_LISTEN_FOR,
retry_listen_for=base.DEFAULT_LISTEN_FOR):
def __init__(
self,
engine,
task_listen_for=base.DEFAULT_LISTEN_FOR,
flow_listen_for=base.DEFAULT_LISTEN_FOR,
retry_listen_for=base.DEFAULT_LISTEN_FOR,
):
super().__init__(
engine, task_listen_for=task_listen_for,
flow_listen_for=flow_listen_for, retry_listen_for=retry_listen_for)
engine,
task_listen_for=task_listen_for,
flow_listen_for=flow_listen_for,
retry_listen_for=retry_listen_for,
)
def _record_atom_event(self, state, atom_name):
meta_update = {'%s-timestamp' % state: time.time()}
@@ -146,8 +163,12 @@ class EventTimeListener(base.Listener):
# Don't let storage failures throw exceptions in a listener method.
self._engine.storage.update_atom_metadata(atom_name, meta_update)
except exc.StorageFailure:
LOG.warning("Failure to store timestamp %s for atom %s",
meta_update, atom_name, exc_info=True)
LOG.warning(
"Failure to store timestamp %s for atom %s",
meta_update,
atom_name,
exc_info=True,
)
def _flow_receiver(self, state, details):
meta_update = {'%s-timestamp' % state: time.time()}
@@ -155,8 +176,12 @@ class EventTimeListener(base.Listener):
# Don't let storage failures throw exceptions in a listener method.
self._engine.storage.update_flow_metadata(meta_update)
except exc.StorageFailure:
LOG.warning("Failure to store timestamp %s for flow %s",
meta_update, details['flow_name'], exc_info=True)
LOG.warning(
"Failure to store timestamp %s for flow %s",
meta_update,
details['flow_name'],
exc_info=True,
)
def _task_receiver(self, state, details):
self._record_atom_event(state, details['task_name'])

3
taskflow/logging.py
View File

@@ -32,12 +32,11 @@ ERROR = logging.ERROR
FATAL = logging.FATAL
INFO = logging.INFO
NOTSET = logging.NOTSET
WARN = logging.WARN
WARN = logging.WARNING
WARNING = logging.WARNING
class _TraceLoggerAdapter(logging.LoggerAdapter):
def trace(self, msg, *args, **kwargs):
"""Delegate a trace call to the underlying logger."""
self.log(TRACE, msg, *args, **kwargs)

79
taskflow/patterns/graph_flow.py
View File

@@ -108,13 +108,23 @@ class Flow(flow.Flow):
if decider is not None:
if not callable(decider):
raise ValueError("Decider boolean callback must be callable")
self._swap(self._link(u, v, manual=True,
decider=decider, decider_depth=decider_depth))
self._swap(
self._link(
u, v, manual=True, decider=decider, decider_depth=decider_depth
)
)
return self
def _link(self, u, v, graph=None,
reason=None, manual=False, decider=None,
decider_depth=None):
def _link(
self,
u,
v,
graph=None,
reason=None,
manual=False,
decider=None,
decider_depth=None,
):
mutable_graph = True
if graph is None:
graph = self._graph
@@ -133,8 +143,10 @@ class Flow(flow.Flow):
pass
if decider_depth is not None:
if decider is None:
raise ValueError("Decider depth requires a decider to be"
" provided along with it")
raise ValueError(
"Decider depth requires a decider to be"
" provided along with it"
)
else:
decider_depth = de.Depth.translate(decider_depth)
attrs[flow.LINK_DECIDER_DEPTH] = decider_depth
@@ -158,10 +170,12 @@ class Flow(flow.Flow):
direct access to the underlying graph).
"""
if not graph.is_directed_acyclic():
raise exc.DependencyFailure("No path through the node(s) in the"
" graph produces an ordering that"
" will allow for logical"
" edge traversal")
raise exc.DependencyFailure(
"No path through the node(s) in the"
" graph produces an ordering that"
" will allow for logical"
" edge traversal"
)
self._graph = graph.freeze()
def add(self, *nodes, **kwargs):
@@ -222,8 +236,9 @@ class Flow(flow.Flow):
provided[value].append(self._retry)
for node in self._graph.nodes:
for value in self._unsatisfied_requires(node, self._graph,
retry_provides):
for value in self._unsatisfied_requires(
node, self._graph, retry_provides
):
required[value].append(node)
for value in node.provides:
provided[value].append(node)
@@ -237,8 +252,9 @@ class Flow(flow.Flow):
# Try to find a valid provider.
if resolve_requires:
for value in self._unsatisfied_requires(node, tmp_graph,
retry_provides):
for value in self._unsatisfied_requires(
node, tmp_graph, retry_provides
):
if value in provided:
providers = provided[value]
if len(providers) > 1:
@@ -248,12 +264,19 @@ class Flow(flow.Flow):
" adding '%(node)s', multiple"
" providers %(providers)s found for"
" required symbol '%(value)s'"
% dict(node=node.name,
providers=sorted(provider_names),
value=value))
% dict(
node=node.name,
providers=sorted(provider_names),
value=value,
)
)
else:
self._link(providers[0], node,
graph=tmp_graph, reason=value)
self._link(
providers[0],
node,
graph=tmp_graph,
reason=value,
)
else:
required[value].append(node)
@@ -266,8 +289,12 @@ class Flow(flow.Flow):
if value in required:
for requiree in list(required[value]):
if requiree is not node:
self._link(node, requiree,
graph=tmp_graph, reason=value)
self._link(
node,
requiree,
graph=tmp_graph,
reason=value,
)
required[value].remove(requiree)
self._swap(tmp_graph)
@@ -305,8 +332,9 @@ class Flow(flow.Flow):
retry_provides.update(self._retry.provides)
g = self._get_subgraph()
for node in g.nodes:
requires.update(self._unsatisfied_requires(node, g,
retry_provides))
requires.update(
self._unsatisfied_requires(node, g, retry_provides)
)
return frozenset(requires)
@@ -365,6 +393,7 @@ class TargetedFlow(Flow):
nodes = [self._target]
nodes.extend(self._graph.bfs_predecessors_iter(self._target))
self._subgraph = gr.DiGraph(
incoming_graph_data=self._graph.subgraph(nodes))
incoming_graph_data=self._graph.subgraph(nodes)
)
self._subgraph.freeze()
return self._subgraph

7
taskflow/patterns/linear_flow.py
View File

@@ -44,8 +44,11 @@ class Flow(flow.Flow):
if not self._graph.has_node(item):
self._graph.add_node(item)
if self._last_item is not self._no_last_item:
self._graph.add_edge(self._last_item, item,
attr_dict={flow.LINK_INVARIANT: True})
self._graph.add_edge(
self._last_item,
item,
attr_dict={flow.LINK_INVARIANT: True},
)
self._last_item = item
return self

11
taskflow/persistence/backends/__init__.py
View File

@@ -56,10 +56,13 @@ def fetch(conf, namespace=BACKEND_NAMESPACE, **kwargs):
backend = backend.split("+", 1)[0]
LOG.debug('Looking for %r backend driver in %r', backend, namespace)
try:
mgr = driver.DriverManager(namespace, backend,
invoke_on_load=True,
invoke_args=(conf,),
invoke_kwds=kwargs)
mgr = driver.DriverManager(
namespace,
backend,
invoke_on_load=True,
invoke_args=(conf,),
invoke_kwds=kwargs,
)
return mgr.driver
except RuntimeError as e:
raise exc.NotFound(f"Could not find backend {backend}: {e}")

26
taskflow/persistence/backends/impl_dir.py
View File

@@ -36,12 +36,13 @@ def _storagefailure_wrapper():
raise
except Exception as e:
if isinstance(e, (IOError, OSError)) and e.errno == errno.ENOENT:
exc.raise_with_cause(exc.NotFound,
'Item not found: %s' % e.filename,
cause=e)
exc.raise_with_cause(
exc.NotFound, 'Item not found: %s' % e.filename, cause=e
)
else:
exc.raise_with_cause(exc.StorageFailure,
"Storage backend internal error", cause=e)
exc.raise_with_cause(
exc.StorageFailure, "Storage backend internal error", cause=e
)
class DirBackend(path_based.PathBasedBackend):
@@ -71,8 +72,9 @@ class DirBackend(path_based.PathBasedBackend):
if max_cache_size is not None:
max_cache_size = int(max_cache_size)
if max_cache_size < 1:
raise ValueError("Maximum cache size must be greater than"
" or equal to one")
raise ValueError(
"Maximum cache size must be greater than or equal to one"
)
self.file_cache = cachetools.LRUCache(max_cache_size)
else:
self.file_cache = {}
@@ -103,8 +105,7 @@ class Connection(path_based.PathBasedConnection):
return cache_info['data']
def _write_to(self, filename, contents):
contents = misc.binary_encode(contents,
encoding=self.backend.encoding)
contents = misc.binary_encode(contents, encoding=self.backend.encoding)
with open(filename, 'wb') as fp:
fp.write(contents)
self.backend.file_cache.pop(filename, None)
@@ -139,8 +140,11 @@ class Connection(path_based.PathBasedConnection):
else:
filter_func = os.path.islink
with _storagefailure_wrapper():
return [child for child in os.listdir(path)
if filter_func(self._join_path(path, child))]
return [
child
for child in os.listdir(path)
if filter_func(self._join_path(path, child))
]
def _ensure_path(self, path):
with _storagefailure_wrapper():

49
taskflow/persistence/backends/impl_memory.py
View File

@@ -73,12 +73,15 @@ class FakeFilesystem:
def normpath(cls, path):
"""Return a normalized absolutized version of the pathname path."""
if not path:
raise ValueError("This filesystem can only normalize paths"
" that are not empty")
raise ValueError(
"This filesystem can only normalize paths that are not empty"
)
if not path.startswith(cls.root_path):
raise ValueError("This filesystem can only normalize"
" paths that start with %s: '%s' is not"
" valid" % (cls.root_path, path))
raise ValueError(
"This filesystem can only normalize"
" paths that start with %s: '%s' is not"
" valid" % (cls.root_path, path)
)
return pp.normpath(path)
#: Split a pathname into a tuple of ``(head, tail)``.
@@ -108,8 +111,7 @@ class FakeFilesystem:
return
node = self._root
for piece in self._iter_pieces(path):
child_node = node.find(piece, only_direct=True,
include_self=False)
child_node = node.find(piece, only_direct=True, include_self=False)
if child_node is None:
child_node = self._insert_child(node, piece)
node = child_node
@@ -154,9 +156,10 @@ class FakeFilesystem:
if links is None:
links = []
if path in links:
raise ValueError("Recursive link following not"
" allowed (loop %s detected)"
% (links + [path]))
raise ValueError(
"Recursive link following not"
" allowed (loop %s detected)" % (links + [path])
)
else:
links.append(path)
return self._get_item(path, links=links)
@@ -186,8 +189,9 @@ class FakeFilesystem:
selector_func = self._metadata_path_selector
else:
selector_func = self._up_to_root_selector
return [selector_func(node, child_node)
for child_node in node.bfs_iter()]
return [
selector_func(node, child_node) for child_node in node.bfs_iter()
]
def ls(self, path, absolute=False):
"""Return list of all children of the given path (not recursive)."""
@@ -197,8 +201,9 @@ class FakeFilesystem:
else:
selector_func = self._up_to_root_selector
child_node_it = iter(node)
return [selector_func(node, child_node)
for child_node in child_node_it]
return [
selector_func(node, child_node) for child_node in child_node_it
]
def clear(self):
"""Remove all nodes (except the root) from this filesystem."""
@@ -219,8 +224,10 @@ class FakeFilesystem:
else:
node_child_count = node.child_count()
if node_child_count:
raise ValueError("Can not delete '%s', it has %s children"
% (path, node_child_count))
raise ValueError(
"Can not delete '%s', it has %s children"
% (path, node_child_count)
)
child_paths = []
if node is self._root:
# Don't drop/pop the root...
@@ -307,8 +314,9 @@ class MemoryBackend(path_based.PathBasedBackend):
def __init__(self, conf=None):
super().__init__(conf)
self.memory = FakeFilesystem(deep_copy=self._conf.get('deep_copy',
True))
self.memory = FakeFilesystem(
deep_copy=self._conf.get('deep_copy', True)
)
self.lock = fasteners.ReaderWriterLock()
def get_connection(self):
@@ -335,8 +343,9 @@ class Connection(path_based.PathBasedConnection):
except exc.TaskFlowException:
raise
except Exception:
exc.raise_with_cause(exc.StorageFailure,
"Storage backend internal error")
exc.raise_with_cause(
exc.StorageFailure, "Storage backend internal error"
)
def _join_path(self, *parts):
return pp.join(*parts)

232
taskflow/persistence/backends/impl_sqlalchemy.py
View File

@@ -109,8 +109,12 @@ DEFAULT_TXN_ISOLATION_LEVELS = {
def _log_statements(log_level, conn, cursor, statement, parameters, *args):
if LOG.isEnabledFor(log_level):
LOG.log(log_level, "Running statement '%s' with parameters %s",
statement, parameters)
LOG.log(
log_level,
"Running statement '%s' with parameters %s",
statement,
parameters,
)
def _in_any(reason, err_haystack):
@@ -188,6 +192,7 @@ class _Alchemist:
NOTE(harlowja): for internal usage only.
"""
def __init__(self, tables):
self._tables = tables
@@ -206,15 +211,17 @@ class _Alchemist:
return atom_cls.from_dict(row)
def atom_query_iter(self, conn, parent_uuid):
q = (sql.select(self._tables.atomdetails).
where(self._tables.atomdetails.c.parent_uuid == parent_uuid))
q = sql.select(self._tables.atomdetails).where(
self._tables.atomdetails.c.parent_uuid == parent_uuid
)
for row in conn.execute(q):
row = row._mapping
yield self.convert_atom_detail(row)
def flow_query_iter(self, conn, parent_uuid):
q = (sql.select(self._tables.flowdetails).
where(self._tables.flowdetails.c.parent_uuid == parent_uuid))
q = sql.select(self._tables.flowdetails).where(
self._tables.flowdetails.c.parent_uuid == parent_uuid
)
for row in conn.execute(q):
row = row._mapping
yield self.convert_flow_detail(row)
@@ -238,6 +245,7 @@ class SQLAlchemyBackend(base.Backend):
"connection": "sqlite:////tmp/test.db",
}
"""
def __init__(self, conf, engine=None):
super().__init__(conf)
if engine is not None:
@@ -275,24 +283,27 @@ class SQLAlchemyBackend(base.Backend):
engine_args["poolclass"] = sa_pool.StaticPool
engine_args["connect_args"] = {'check_same_thread': False}
else:
for (k, lookup_key) in [('pool_size', 'max_pool_size'),
('max_overflow', 'max_overflow'),
('pool_timeout', 'pool_timeout')]:
for k, lookup_key in [
('pool_size', 'max_pool_size'),
('max_overflow', 'max_overflow'),
('pool_timeout', 'pool_timeout'),
]:
if lookup_key in conf:
engine_args[k] = misc.as_int(conf.pop(lookup_key))
if 'isolation_level' not in conf:
# Check driver name exact matches first, then try driver name
# partial matches...
txn_isolation_levels = conf.pop('isolation_levels',
DEFAULT_TXN_ISOLATION_LEVELS)
txn_isolation_levels = conf.pop(
'isolation_levels', DEFAULT_TXN_ISOLATION_LEVELS
)
level_applied = False
for (driver, level) in txn_isolation_levels.items():
for driver, level in txn_isolation_levels.items():
if driver == e_url.drivername:
engine_args['isolation_level'] = level
level_applied = True
break
if not level_applied:
for (driver, level) in txn_isolation_levels.items():
for driver, level in txn_isolation_levels.items():
if e_url.drivername.find(driver) != -1:
engine_args['isolation_level'] = level
break
@@ -304,13 +315,17 @@ class SQLAlchemyBackend(base.Backend):
engine = sa.create_engine(sql_connection, **engine_args)
log_statements = conf.pop('log_statements', False)
if _as_bool(log_statements):
log_statements_level = conf.pop("log_statements_level",
logging.TRACE)
sa.event.listen(engine, "before_cursor_execute",
functools.partial(_log_statements,
log_statements_level))
checkin_yield = conf.pop('checkin_yield',
eventlet_utils.EVENTLET_AVAILABLE)
log_statements_level = conf.pop(
"log_statements_level", logging.TRACE
)
sa.event.listen(
engine,
"before_cursor_execute",
functools.partial(_log_statements, log_statements_level),
)
checkin_yield = conf.pop(
'checkin_yield', eventlet_utils.EVENTLET_AVAILABLE
)
if _as_bool(checkin_yield):
sa.event.listen(engine, 'checkin', _thread_yield)
if 'mysql' in e_url.drivername:
@@ -320,8 +335,9 @@ class SQLAlchemyBackend(base.Backend):
if 'mysql_sql_mode' in conf:
mode = conf.pop('mysql_sql_mode')
if mode is not None:
sa.event.listen(engine, 'connect',
functools.partial(_set_sql_mode, mode))
sa.event.listen(
engine, 'connect', functools.partial(_set_sql_mode, mode)
)
return engine
@property
@@ -362,13 +378,19 @@ class Connection(base.Connection):
def _retry_on_exception(exc):
LOG.warning("Engine connection (validate) failed due to '%s'", exc)
if isinstance(exc, sa_exc.OperationalError) and \
_is_db_connection_error(str(exc.args[0])):
if isinstance(
exc, sa_exc.OperationalError
) and _is_db_connection_error(str(exc.args[0])):
# We may be able to fix this by retrying...
return True
if isinstance(exc, (sa_exc.TimeoutError,
sa_exc.ResourceClosedError,
sa_exc.DisconnectionError)):
if isinstance(
exc,
(
sa_exc.TimeoutError,
sa_exc.ResourceClosedError,
sa_exc.DisconnectionError,
),
):
# We may be able to fix this by retrying...
return True
# Other failures we likely can't fix by retrying...
@@ -378,7 +400,7 @@ class Connection(base.Connection):
stop=tenacity.stop_after_attempt(max(0, int(max_retries))),
wait=tenacity.wait_exponential(),
reraise=True,
retry=tenacity.retry_if_exception(_retry_on_exception)
retry=tenacity.retry_if_exception(_retry_on_exception),
)
def _try_connect(engine):
# See if we can make a connection happen.
@@ -408,8 +430,9 @@ class Connection(base.Connection):
else:
migration.db_sync(conn)
except sa_exc.SQLAlchemyError:
exc.raise_with_cause(exc.StorageFailure,
"Failed upgrading database version")
exc.raise_with_cause(
exc.StorageFailure, "Failed upgrading database version"
)
def clear_all(self):
try:
@@ -417,27 +440,33 @@ class Connection(base.Connection):
with self._engine.begin() as conn:
conn.execute(logbooks.delete())
except sa_exc.DBAPIError:
exc.raise_with_cause(exc.StorageFailure,
"Failed clearing all entries")
exc.raise_with_cause(
exc.StorageFailure, "Failed clearing all entries"
)
def update_atom_details(self, atom_detail):
try:
atomdetails = self._tables.atomdetails
with self._engine.begin() as conn:
q = (sql.select(atomdetails).
where(atomdetails.c.uuid == atom_detail.uuid))
q = sql.select(atomdetails).where(
atomdetails.c.uuid == atom_detail.uuid
)
row = conn.execute(q).first()
if not row:
raise exc.NotFound("No atom details found with uuid"
" '%s'" % atom_detail.uuid)
raise exc.NotFound(
"No atom details found with uuid"
" '%s'" % atom_detail.uuid
)
row = row._mapping
e_ad = self._converter.convert_atom_detail(row)
self._update_atom_details(conn, atom_detail, e_ad)
return e_ad
except sa_exc.SQLAlchemyError:
exc.raise_with_cause(exc.StorageFailure,
"Failed updating atom details"
" with uuid '%s'" % atom_detail.uuid)
exc.raise_with_cause(
exc.StorageFailure,
"Failed updating atom details"
" with uuid '%s'" % atom_detail.uuid,
)
def _insert_flow_details(self, conn, fd, parent_uuid):
value = fd.to_dict()
@@ -454,15 +483,19 @@ class Connection(base.Connection):
def _update_atom_details(self, conn, ad, e_ad):
e_ad.merge(ad)
conn.execute(sql.update(self._tables.atomdetails)
.where(self._tables.atomdetails.c.uuid == e_ad.uuid)
.values(e_ad.to_dict()))
conn.execute(
sql.update(self._tables.atomdetails)
.where(self._tables.atomdetails.c.uuid == e_ad.uuid)
.values(e_ad.to_dict())
)
def _update_flow_details(self, conn, fd, e_fd):
e_fd.merge(fd)
conn.execute(sql.update(self._tables.flowdetails)
.where(self._tables.flowdetails.c.uuid == e_fd.uuid)
.values(e_fd.to_dict()))
conn.execute(
sql.update(self._tables.flowdetails)
.where(self._tables.flowdetails.c.uuid == e_fd.uuid)
.values(e_fd.to_dict())
)
for ad in fd:
e_ad = e_fd.find(ad.uuid)
if e_ad is None:
@@ -475,21 +508,26 @@ class Connection(base.Connection):
try:
flowdetails = self._tables.flowdetails
with self._engine.begin() as conn:
q = (sql.select(flowdetails).
where(flowdetails.c.uuid == flow_detail.uuid))
q = sql.select(flowdetails).where(
flowdetails.c.uuid == flow_detail.uuid
)
row = conn.execute(q).first()
if not row:
raise exc.NotFound("No flow details found with"
" uuid '%s'" % flow_detail.uuid)
raise exc.NotFound(
"No flow details found with"
" uuid '%s'" % flow_detail.uuid
)
row = row._mapping
e_fd = self._converter.convert_flow_detail(row)
self._converter.populate_flow_detail(conn, e_fd)
self._update_flow_details(conn, flow_detail, e_fd)
return e_fd
except sa_exc.SQLAlchemyError:
exc.raise_with_cause(exc.StorageFailure,
"Failed updating flow details with"
" uuid '%s'" % flow_detail.uuid)
exc.raise_with_cause(
exc.StorageFailure,
"Failed updating flow details with"
" uuid '%s'" % flow_detail.uuid,
)
def destroy_logbook(self, book_uuid):
try:
@@ -498,27 +536,31 @@ class Connection(base.Connection):
q = logbooks.delete().where(logbooks.c.uuid == book_uuid)
r = conn.execute(q)
if r.rowcount == 0:
raise exc.NotFound("No logbook found with"
" uuid '%s'" % book_uuid)
raise exc.NotFound(
"No logbook found with uuid '%s'" % book_uuid
)
except sa_exc.DBAPIError:
exc.raise_with_cause(exc.StorageFailure,
"Failed destroying logbook '%s'" % book_uuid)
exc.raise_with_cause(
exc.StorageFailure,
"Failed destroying logbook '%s'" % book_uuid,
)
def save_logbook(self, book):
try:
logbooks = self._tables.logbooks
with self._engine.begin() as conn:
q = (sql.select(logbooks).
where(logbooks.c.uuid == book.uuid))
q = sql.select(logbooks).where(logbooks.c.uuid == book.uuid)
row = conn.execute(q).first()
if row:
row = row._mapping
e_lb = self._converter.convert_book(row)
self._converter.populate_book(conn, e_lb)
e_lb.merge(book)
conn.execute(sql.update(logbooks)
.where(logbooks.c.uuid == e_lb.uuid)
.values(e_lb.to_dict()))
conn.execute(
sql.update(logbooks)
.where(logbooks.c.uuid == e_lb.uuid)
.values(e_lb.to_dict())
)
for fd in book:
e_fd = e_lb.find(fd.uuid)
if e_fd is None:
@@ -534,27 +576,28 @@ class Connection(base.Connection):
return book
except sa_exc.DBAPIError:
exc.raise_with_cause(
exc.StorageFailure,
"Failed saving logbook '%s'" % book.uuid)
exc.StorageFailure, "Failed saving logbook '%s'" % book.uuid
)
def get_logbook(self, book_uuid, lazy=False):
try:
logbooks = self._tables.logbooks
with self._engine.connect() as conn:
q = (sql.select(logbooks).
where(logbooks.c.uuid == book_uuid))
q = sql.select(logbooks).where(logbooks.c.uuid == book_uuid)
row = conn.execute(q).first()
if not row:
raise exc.NotFound("No logbook found with"
" uuid '%s'" % book_uuid)
raise exc.NotFound(
"No logbook found with uuid '%s'" % book_uuid
)
row = row._mapping
book = self._converter.convert_book(row)
if not lazy:
self._converter.populate_book(conn, book)
return book
except sa_exc.DBAPIError:
exc.raise_with_cause(exc.StorageFailure,
"Failed getting logbook '%s'" % book_uuid)
exc.raise_with_cause(
exc.StorageFailure, "Failed getting logbook '%s'" % book_uuid
)
def get_logbooks(self, lazy=False):
gathered = []
@@ -568,8 +611,7 @@ class Connection(base.Connection):
self._converter.populate_book(conn, book)
gathered.append(book)
except sa_exc.DBAPIError:
exc.raise_with_cause(exc.StorageFailure,
"Failed getting logbooks")
exc.raise_with_cause(exc.StorageFailure, "Failed getting logbooks")
for book in gathered:
yield book
@@ -582,47 +624,54 @@ class Connection(base.Connection):
self._converter.populate_flow_detail(conn, fd)
gathered.append(fd)
except sa_exc.DBAPIError:
exc.raise_with_cause(exc.StorageFailure,
"Failed getting flow details in"
" logbook '%s'" % book_uuid)
exc.raise_with_cause(
exc.StorageFailure,
"Failed getting flow details in logbook '%s'" % book_uuid,
)
yield from gathered
def get_flow_details(self, fd_uuid, lazy=False):
try:
flowdetails = self._tables.flowdetails
with self._engine.begin() as conn:
q = (sql.select(flowdetails).
where(flowdetails.c.uuid == fd_uuid))
q = sql.select(flowdetails).where(
flowdetails.c.uuid == fd_uuid
)
row = conn.execute(q).first()
if not row:
raise exc.NotFound("No flow details found with uuid"
" '%s'" % fd_uuid)
raise exc.NotFound(
"No flow details found with uuid '%s'" % fd_uuid
)
row = row._mapping
fd = self._converter.convert_flow_detail(row)
if not lazy:
self._converter.populate_flow_detail(conn, fd)
return fd
except sa_exc.SQLAlchemyError:
exc.raise_with_cause(exc.StorageFailure,
"Failed getting flow details with"
" uuid '%s'" % fd_uuid)
exc.raise_with_cause(
exc.StorageFailure,
"Failed getting flow details with uuid '%s'" % fd_uuid,
)
def get_atom_details(self, ad_uuid):
try:
atomdetails = self._tables.atomdetails
with self._engine.begin() as conn:
q = (sql.select(atomdetails).
where(atomdetails.c.uuid == ad_uuid))
q = sql.select(atomdetails).where(
atomdetails.c.uuid == ad_uuid
)
row = conn.execute(q).first()
if not row:
raise exc.NotFound("No atom details found with uuid"
" '%s'" % ad_uuid)
raise exc.NotFound(
"No atom details found with uuid '%s'" % ad_uuid
)
row = row._mapping
return self._converter.convert_atom_detail(row)
except sa_exc.SQLAlchemyError:
exc.raise_with_cause(exc.StorageFailure,
"Failed getting atom details with"
" uuid '%s'" % ad_uuid)
exc.raise_with_cause(
exc.StorageFailure,
"Failed getting atom details with uuid '%s'" % ad_uuid,
)
def get_atoms_for_flow(self, fd_uuid):
gathered = []
@@ -631,9 +680,10 @@ class Connection(base.Connection):
for ad in self._converter.atom_query_iter(conn, fd_uuid):
gathered.append(ad)
except sa_exc.DBAPIError:
exc.raise_with_cause(exc.StorageFailure,
"Failed getting atom details in flow"
" detail '%s'" % fd_uuid)
exc.raise_with_cause(
exc.StorageFailure,
"Failed getting atom details in flow detail '%s'" % fd_uuid,
)
yield from gathered
def close(self):

37
taskflow/persistence/backends/impl_zookeeper.py
View File

@@ -79,8 +79,9 @@ class ZkBackend(path_based.PathBasedBackend):
try:
k_utils.finalize_client(self._client)
except (k_exc.KazooException, k_exc.ZookeeperError):
exc.raise_with_cause(exc.StorageFailure,
"Unable to finalize client")
exc.raise_with_cause(
exc.StorageFailure, "Unable to finalize client"
)
class ZkConnection(path_based.PathBasedConnection):
@@ -103,20 +104,23 @@ class ZkConnection(path_based.PathBasedConnection):
try:
yield
except self._client.handler.timeout_exception:
exc.raise_with_cause(exc.StorageFailure,
"Storage backend timeout")
exc.raise_with_cause(exc.StorageFailure, "Storage backend timeout")
except k_exc.SessionExpiredError:
exc.raise_with_cause(exc.StorageFailure,
"Storage backend session has expired")
exc.raise_with_cause(
exc.StorageFailure, "Storage backend session has expired"
)
except k_exc.NoNodeError:
exc.raise_with_cause(exc.NotFound,
"Storage backend node not found")
exc.raise_with_cause(
exc.NotFound, "Storage backend node not found"
)
except k_exc.NodeExistsError:
exc.raise_with_cause(exc.Duplicate,
"Storage backend duplicate node")
exc.raise_with_cause(
exc.Duplicate, "Storage backend duplicate node"
)
except (k_exc.KazooException, k_exc.ZookeeperError):
exc.raise_with_cause(exc.StorageFailure,
"Storage backend internal error")
exc.raise_with_cause(
exc.StorageFailure, "Storage backend internal error"
)
def _join_path(self, *parts):
return paths.join(*parts)
@@ -161,8 +165,11 @@ class ZkConnection(path_based.PathBasedConnection):
with self._exc_wrapper():
try:
if strutils.bool_from_string(
self._conf.get('check_compatible'), default=True):
self._conf.get('check_compatible'), default=True
):
k_utils.check_compatible(self._client, MIN_ZK_VERSION)
except exc.IncompatibleVersion:
exc.raise_with_cause(exc.StorageFailure, "Backend storage is"
" not a compatible version")
exc.raise_with_cause(
exc.StorageFailure,
"Backend storage is not a compatible version",
)

14
taskflow/persistence/backends/sqlalchemy/alembic/env.py
View File

@@ -60,19 +60,23 @@ def run_migrations_online():
if connectable is None:
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix='sqlalchemy.', poolclass=pool.NullPool)
prefix='sqlalchemy.',
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(connection=connection,
target_metadata=target_metadata)
context.configure(
connection=connection, target_metadata=target_metadata
)
with context.begin_transaction():
context.run_migrations()
else:
context.configure(
connection=connectable,
target_metadata=target_metadata)
connection=connectable, target_metadata=target_metadata
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:

Some files were not shown because too many files have changed in this diff Show More