ae9c701f90
This adds a executor backed job dispatching base class and has the existing blocking executor use it by running jobs and dispatching jobs into a sync executor. It also allows for dispatching jobs into a thread executor, or other executor via a new '_executor_factory' method that can generate executors (it can be overriden in the non-blocking executor to provide your own executors instances). This does alter the behavior in that now that jobs are dispatched into an executor we no longer can immediatly know if a job was dispatched and raised an exception or whether it will raise an exception in the future, so we now alter the 'local_dispatched' to just be a boolean that is used to determine if any dispatches happened (failure or not). Change-Id: I485770e8f4c85d3833892a453c9fb5168d8f0407
70 lines
3.0 KiB
Python
70 lines
3.0 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import futurist
|
|
import six
|
|
|
|
from taskflow.conductors.backends import impl_executor
|
|
from taskflow.utils import threading_utils as tu
|
|
|
|
|
|
class NonBlockingConductor(impl_executor.ExecutorConductor):
|
|
"""Non-blocking conductor that processes job(s) using a thread executor.
|
|
|
|
NOTE(harlowja): A custom executor factory can be provided via keyword
|
|
argument ``executor_factory``, if provided it will be
|
|
invoked at
|
|
:py:meth:`~taskflow.conductors.base.Conductor.run` time
|
|
with one positional argument (this conductor) and it must
|
|
return a compatible `executor`_ which can be used
|
|
to submit jobs to. If ``None`` is a provided a thread pool
|
|
backed executor is selected by default (it will have
|
|
an equivalent number of workers as this conductors
|
|
simultaneous job count).
|
|
|
|
.. _executor: https://docs.python.org/dev/library/\
|
|
concurrent.futures.html#executor-objects
|
|
"""
|
|
|
|
MAX_SIMULTANEOUS_JOBS = tu.get_optimal_thread_count()
|
|
"""
|
|
Default maximum number of jobs that can be in progress at the same time.
|
|
"""
|
|
|
|
def _default_executor_factory(self):
|
|
max_simultaneous_jobs = self._max_simultaneous_jobs
|
|
if max_simultaneous_jobs <= 0:
|
|
max_workers = tu.get_optimal_thread_count()
|
|
else:
|
|
max_workers = max_simultaneous_jobs
|
|
return futurist.ThreadPoolExecutor(max_workers=max_workers)
|
|
|
|
def __init__(self, name, jobboard,
|
|
persistence=None, engine=None,
|
|
engine_options=None, wait_timeout=None,
|
|
log=None, max_simultaneous_jobs=MAX_SIMULTANEOUS_JOBS,
|
|
executor_factory=None):
|
|
super(NonBlockingConductor, self).__init__(
|
|
name, jobboard,
|
|
persistence=persistence, engine=engine,
|
|
engine_options=engine_options, wait_timeout=wait_timeout,
|
|
log=log, max_simultaneous_jobs=max_simultaneous_jobs)
|
|
if executor_factory is None:
|
|
self._executor_factory = self._default_executor_factory
|
|
else:
|
|
if not six.callable(executor_factory):
|
|
raise ValueError("Provided keyword argument 'executor_factory'"
|
|
" must be callable")
|
|
self._executor_factory = executor_factory
|