Allow for specifying green threaded to parallel engine
Currently when a string is passed to the parallel engine it will only know how to create a process or a native thread based executor. The futurist library also supports making a green thread based executor, so support creating it. This will save glance some code that they have to create a executor based on different options (one of those is a green option). Change-Id: I15c164a38b4445d28eb6062aed6c56cce0e0364b
This commit is contained in:
parent
1bc8dd9bca
commit
261826674b
|
@ -519,6 +519,12 @@ String (case insensitive) Executor used
|
||||||
``thread`` :class:`~.executor.ParallelThreadTaskExecutor`
|
``thread`` :class:`~.executor.ParallelThreadTaskExecutor`
|
||||||
``threaded`` :class:`~.executor.ParallelThreadTaskExecutor`
|
``threaded`` :class:`~.executor.ParallelThreadTaskExecutor`
|
||||||
``threads`` :class:`~.executor.ParallelThreadTaskExecutor`
|
``threads`` :class:`~.executor.ParallelThreadTaskExecutor`
|
||||||
|
``greenthread`` :class:`~.executor.ParallelThreadTaskExecutor`
|
||||||
|
(greened version)
|
||||||
|
``greedthreaded`` :class:`~.executor.ParallelThreadTaskExecutor`
|
||||||
|
(greened version)
|
||||||
|
``greenthreads`` :class:`~.executor.ParallelThreadTaskExecutor`
|
||||||
|
(greened version)
|
||||||
=========================== ===============================================
|
=========================== ===============================================
|
||||||
|
|
||||||
* ``max_workers``: a integer that will affect the number of parallel
|
* ``max_workers``: a integer that will affect the number of parallel
|
||||||
|
@ -562,6 +568,9 @@ String (case insensitive) Executor used
|
||||||
executor.ParallelProcessTaskExecutor),
|
executor.ParallelProcessTaskExecutor),
|
||||||
_ExecutorTextMatch(frozenset(['thread', 'threads', 'threaded']),
|
_ExecutorTextMatch(frozenset(['thread', 'threads', 'threaded']),
|
||||||
executor.ParallelThreadTaskExecutor),
|
executor.ParallelThreadTaskExecutor),
|
||||||
|
_ExecutorTextMatch(frozenset(['greenthread', 'greenthreads',
|
||||||
|
'greenthreaded']),
|
||||||
|
executor.ParallelGreenThreadTaskExecutor),
|
||||||
]
|
]
|
||||||
|
|
||||||
# Used when no executor is provided (either a string or object)...
|
# Used when no executor is provided (either a string or object)...
|
||||||
|
|
|
@ -476,6 +476,24 @@ class ParallelThreadTaskExecutor(ParallelTaskExecutor):
|
||||||
return futurist.ThreadPoolExecutor(max_workers=max_workers)
|
return futurist.ThreadPoolExecutor(max_workers=max_workers)
|
||||||
|
|
||||||
|
|
||||||
|
class ParallelGreenThreadTaskExecutor(ParallelThreadTaskExecutor):
|
||||||
|
"""Executes tasks in parallel using a greenthread pool executor."""
|
||||||
|
|
||||||
|
DEFAULT_WORKERS = 1000
|
||||||
|
"""
|
||||||
|
Default number of workers when ``None`` is passed; being that
|
||||||
|
greenthreads don't map to native threads or processors very well this
|
||||||
|
is more of a guess/somewhat arbitrary, but it does match what the eventlet
|
||||||
|
greenpool default size is (so at least it's consistent with what eventlet
|
||||||
|
does).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _create_executor(self, max_workers=None):
|
||||||
|
if max_workers is None:
|
||||||
|
max_workers = self.DEFAULT_WORKERS
|
||||||
|
return futurist.GreenThreadPoolExecutor(max_workers=max_workers)
|
||||||
|
|
||||||
|
|
||||||
class ParallelProcessTaskExecutor(ParallelTaskExecutor):
|
class ParallelProcessTaskExecutor(ParallelTaskExecutor):
|
||||||
"""Executes tasks in parallel using a process pool executor.
|
"""Executes tasks in parallel using a process pool executor.
|
||||||
|
|
||||||
|
|
|
@ -25,8 +25,6 @@ top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
|
||||||
os.pardir))
|
os.pardir))
|
||||||
sys.path.insert(0, top_dir)
|
sys.path.insert(0, top_dir)
|
||||||
|
|
||||||
import futurist
|
|
||||||
|
|
||||||
from taskflow import engines
|
from taskflow import engines
|
||||||
from taskflow.patterns import linear_flow as lf
|
from taskflow.patterns import linear_flow as lf
|
||||||
from taskflow.patterns import unordered_flow as uf
|
from taskflow.patterns import unordered_flow as uf
|
||||||
|
@ -82,35 +80,29 @@ song.add(PrinterTask("conductor@begin",
|
||||||
|
|
||||||
# Run in parallel using eventlet green threads...
|
# Run in parallel using eventlet green threads...
|
||||||
try:
|
try:
|
||||||
executor = futurist.GreenThreadPoolExecutor()
|
import eventlet as _eventlet # noqa
|
||||||
except RuntimeError:
|
except ImportError:
|
||||||
# No eventlet currently active, skip running with it...
|
# No eventlet currently active, skip running with it...
|
||||||
pass
|
pass
|
||||||
else:
|
else:
|
||||||
print("-- Running in parallel using eventlet --")
|
print("-- Running in parallel using eventlet --")
|
||||||
with executor:
|
e = engines.load(song, executor='greenthreaded', engine='parallel',
|
||||||
e = engines.load(song, executor=executor, engine='parallel')
|
max_workers=1)
|
||||||
e.run()
|
e.run()
|
||||||
print("-- Statistics gathered --")
|
|
||||||
print(e.statistics)
|
|
||||||
|
|
||||||
|
|
||||||
# Run in parallel using real threads...
|
# Run in parallel using real threads...
|
||||||
with futurist.ThreadPoolExecutor(max_workers=1) as executor:
|
print("-- Running in parallel using threads --")
|
||||||
print("-- Running in parallel using threads --")
|
e = engines.load(song, executor='threaded', engine='parallel',
|
||||||
e = engines.load(song, executor=executor, engine='parallel')
|
max_workers=1)
|
||||||
e.run()
|
e.run()
|
||||||
print("-- Statistics gathered --")
|
|
||||||
print(e.statistics)
|
|
||||||
|
|
||||||
|
|
||||||
# Run in parallel using external processes...
|
# Run in parallel using external processes...
|
||||||
with futurist.ProcessPoolExecutor(max_workers=1) as executor:
|
print("-- Running in parallel using processes --")
|
||||||
print("-- Running in parallel using processes --")
|
e = engines.load(song, executor='processes', engine='parallel',
|
||||||
e = engines.load(song, executor=executor, engine='parallel')
|
max_workers=1)
|
||||||
e.run()
|
e.run()
|
||||||
print("-- Statistics gathered --")
|
|
||||||
print(e.statistics)
|
|
||||||
|
|
||||||
|
|
||||||
# Run serially (aka, if the workflow could have been ran in parallel, it will
|
# Run serially (aka, if the workflow could have been ran in parallel, it will
|
||||||
|
|
|
@ -1449,8 +1449,7 @@ class ParallelEngineWithEventletTest(EngineTaskTest,
|
||||||
flow_detail=None, executor=None, store=None,
|
flow_detail=None, executor=None, store=None,
|
||||||
**kwargs):
|
**kwargs):
|
||||||
if executor is None:
|
if executor is None:
|
||||||
executor = futurist.GreenThreadPoolExecutor()
|
executor = 'greenthreads'
|
||||||
self.addCleanup(executor.shutdown)
|
|
||||||
return taskflow.engines.load(flow, flow_detail=flow_detail,
|
return taskflow.engines.load(flow, flow_detail=flow_detail,
|
||||||
backend=self.backend, engine='parallel',
|
backend=self.backend, engine='parallel',
|
||||||
executor=executor,
|
executor=executor,
|
||||||
|
|
|
@ -14,7 +14,6 @@
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
import futurist
|
|
||||||
import testtools
|
import testtools
|
||||||
|
|
||||||
import taskflow.engines
|
import taskflow.engines
|
||||||
|
@ -1305,8 +1304,7 @@ class ParallelEngineWithEventletTest(RetryTest, test.TestCase):
|
||||||
def _make_engine(self, flow, defer_reverts=None, flow_detail=None,
|
def _make_engine(self, flow, defer_reverts=None, flow_detail=None,
|
||||||
executor=None):
|
executor=None):
|
||||||
if executor is None:
|
if executor is None:
|
||||||
executor = futurist.GreenThreadPoolExecutor()
|
executor = 'greenthreads'
|
||||||
self.addCleanup(executor.shutdown)
|
|
||||||
return taskflow.engines.load(flow,
|
return taskflow.engines.load(flow,
|
||||||
flow_detail=flow_detail,
|
flow_detail=flow_detail,
|
||||||
backend=self.backend,
|
backend=self.backend,
|
||||||
|
|
Loading…
Reference in New Issue