b52db18522
To enable a parallel process executor to be used without having to pass in a futures executor allow for the executor option to be a string 'processes' (or similar) that will cause the default parallel process executor to be used automatically. Also allow for a 'threads' string that ensure a parallel thread executor is used to match the ability to uses processes. This also adjusts the WBE engine to have a similar executor fetching function (which in the WBE case now validates a provided executor to be of the desired type). Change-Id: I54a82584c32c697922507b4f6e01ea7b8acc73c6
74 lines
3.4 KiB
Python
74 lines
3.4 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
from taskflow.engines.action_engine import engine
|
|
from taskflow.engines.worker_based import executor
|
|
from taskflow.engines.worker_based import protocol as pr
|
|
from taskflow import storage as t_storage
|
|
|
|
|
|
class WorkerBasedActionEngine(engine.ActionEngine):
|
|
"""Worker based action engine.
|
|
|
|
Specific backend options:
|
|
|
|
:param exchange: broker exchange exchange name in which executor / worker
|
|
communication is performed
|
|
:param url: broker connection url (see format in kombu documentation)
|
|
:param topics: list of workers topics to communicate with (this will also
|
|
be learned by listening to the notifications that workers
|
|
emit).
|
|
:param transport: transport to be used (e.g. amqp, memory, etc.)
|
|
:param transport_options: transport specific options
|
|
:param transition_timeout: numeric value (or None for infinite) to wait
|
|
for submitted remote requests to transition out
|
|
of the (PENDING, WAITING) request states. When
|
|
expired the associated task the request was made
|
|
for will have its result become a
|
|
`RequestTimeout` exception instead of its
|
|
normally returned value (or raised exception).
|
|
"""
|
|
|
|
_storage_factory = t_storage.SingleThreadedStorage
|
|
|
|
def __init__(self, flow, flow_detail, backend, options):
|
|
super(WorkerBasedActionEngine, self).__init__(flow, flow_detail,
|
|
backend, options)
|
|
# This ensures that any provided executor will be validated before
|
|
# we get to far in the compilation/execution pipeline...
|
|
self._task_executor = self._fetch_task_executor(self._options,
|
|
self._flow_detail)
|
|
|
|
@classmethod
|
|
def _fetch_task_executor(cls, options, flow_detail):
|
|
try:
|
|
e = options['executor']
|
|
if not isinstance(e, executor.WorkerTaskExecutor):
|
|
raise TypeError("Expected an instance of type '%s' instead of"
|
|
" type '%s' for 'executor' option"
|
|
% (executor.WorkerTaskExecutor, type(e)))
|
|
return e
|
|
except KeyError:
|
|
return executor.WorkerTaskExecutor(
|
|
uuid=flow_detail.uuid,
|
|
url=options.get('url'),
|
|
exchange=options.get('exchange', 'default'),
|
|
topics=options.get('topics', []),
|
|
transport=options.get('transport'),
|
|
transport_options=options.get('transport_options'),
|
|
transition_timeout=options.get('transition_timeout',
|
|
pr.REQUEST_TIMEOUT))
|