diff --git a/README.rst b/README.rst index 685f35a1..332a4b69 100644 --- a/README.rst +++ b/README.rst @@ -37,6 +37,14 @@ To run Mistral Task Executor instance perform the following command in a shell: Note that at least one Engine instance and one Executor instance should be running so that workflow tasks are processed by Mistral. +Running Multiple Mistral Servers Under the Same Process +------------------------------------------------------- +To run more than one server (API, Engine, or Task Executor) on the same process, perform the following command in a shell: + + tox -evenv -- python mistral/cmd/launch.py --server api,engine --config-file path_to_config + +The --server command line option can be a comma delimited list. The valid options are "all" (by default if not specified) or any combination of "api", "engine", and "executor". It's important to note that the "fake" transport for the rpc_backend defined in the config file should only be used if "all" the Mistral servers are launched on the same process. Otherwise, messages do not get delivered if the Mistral servers are launched on different processes because the "fake" transport is using an in process queue. + Debugging --------- diff --git a/etc/mistral.conf.example b/etc/mistral.conf.example index 2b1861fb..547d6cb8 100644 --- a/etc/mistral.conf.example +++ b/etc/mistral.conf.example @@ -25,8 +25,9 @@ rabbit_password = guest # Options for oslo.messaging #rpc_backend=rabbit -# Specifies which mistral server to start by the launch script. (string value) -# Choices are all, api, engine, and executor. +# Specifies which mistral server(s) to start by the launch script. (string value) +# Choices are all (default) or comma delimited list of api, engine, and/or executor. +# Example: server=api,engine #server=all [api] diff --git a/mistral/cmd/launch.py b/mistral/cmd/launch.py index 223dbd4d..a54c99b8 100755 --- a/mistral/cmd/launch.py +++ b/mistral/cmd/launch.py @@ -53,20 +53,20 @@ def launch_executor(transport): # Since engine and executor are tightly coupled, use the engine # configuration to decide which executor to get. endpoints = [executor.get_executor(cfg.CONF.engine.engine, transport)] - ex_server = messaging.get_rpc_server( + server = messaging.get_rpc_server( transport, target, endpoints, executor='eventlet') - ex_server.start() - ex_server.wait() + server.start() + server.wait() def launch_engine(transport): target = messaging.Target(topic=cfg.CONF.engine.topic, server=cfg.CONF.engine.host) endpoints = [engine.get_engine(cfg.CONF.engine.engine, transport)] - en_server = messaging.get_rpc_server( + server = messaging.get_rpc_server( transport, target, endpoints, executor='eventlet') - en_server.start() - en_server.wait() + server.start() + server.wait() def launch_api(transport): @@ -79,12 +79,20 @@ def launch_api(transport): server.serve_forever() -def launch_all(transport): +def launch_any(transport, options): # Launch the servers on different threads. - t1 = eventlet.spawn(launch_executor, transport) - t2 = eventlet.spawn(launch_engine, transport) - t3 = eventlet.spawn(launch_api, transport) - t1.wait() and t2.wait() and t3.wait() + threads = [eventlet.spawn(LAUNCH_OPTIONS[option], transport) + for option in options] + [thread.wait() for thread in threads] + + +# Map cli options to appropriate functions. The cli options are +# registered in mistral's config.py. +LAUNCH_OPTIONS = { + 'api': launch_api, + 'engine': launch_engine, + 'executor': launch_executor +} def main(): @@ -92,31 +100,34 @@ def main(): config.parse_args() logging.setup('Mistral') - # Map cli options to appropriate functions. The cli options are - # registered in mistral's config.py. - launch_options = { - 'all': launch_all, - 'api': launch_api, - 'engine': launch_engine, - 'executor': launch_executor - } - # Please refer to the oslo.messaging documentation for transport - # configuration. The default transport for oslo.messaging is rabbitMQ. - # The available transport drivers are listed under oslo.messaging at - # ./oslo/messaging/rpc/_drivers. The drivers are prefixed with "impl". - # The transport driver is specified using the rpc_backend option in the - # default section of the oslo configuration file. The expected value - # for rpc_backend is the last part of the driver name. For example, - # the driver for rabbit is impl_rabbit and for the fake driver is - # impl_fake. The rpc_backend value for these are "rabbit" and "fake" - # respectively. There are additional options such as ssl and credential - # that can be specified depending on the driver. Please refer to the - # driver implementation for those additional options. + # configuration. The default transport for oslo.messaging is + # rabbitMQ. The available transport drivers are listed in the + # setup.cfg file in oslo.messaging under the entry_points section for + # oslo.messaging.drivers. The transport driver is specified using the + # rpc_backend option in the default section of the oslo configuration + # file. The expected value for the rpc_backend is one of the key + # values available for the oslo.messaging.drivers (i.e. rabbit, fake). + # There are additional options such as ssl and credential that can be + # specified depending on the driver. Please refer to the driver + # implementation for those additional options. It's important to note + # that the "fake" transport should only be used if "all" the Mistral + # servers are launched on the same process. Otherwise, messages do not + # get delivered if the Mistral servers are launched on different + # processes because the "fake" transport is using an in process queue. transport = messaging.get_transport(cfg.CONF) - # Launch server(s). - launch_options[cfg.CONF.server](transport) + if (cfg.CONF.server == ['all']): + # Launch all servers. + launch_any(transport, LAUNCH_OPTIONS.keys()) + else: + # Validate launch option. + if set(cfg.CONF.server) - set(LAUNCH_OPTIONS.keys()): + raise Exception('Valid options are all or any combination of ' + 'api, engine, and executor.') + + # Launch distinct set of server(s). + launch_any(transport, set(cfg.CONF.server)) except RuntimeError, e: sys.stderr.write("ERROR: %s\n" % e) diff --git a/mistral/config.py b/mistral/config.py index 80ccb000..8265dfa1 100644 --- a/mistral/config.py +++ b/mistral/config.py @@ -86,11 +86,12 @@ executor_opts = [ help='The version of the executor.') ] -launch_opt = cfg.StrOpt( +launch_opt = cfg.ListOpt( 'server', - default='all', - choices=('all', 'api', 'engine', 'executor'), - help='Specifies which mistral server to start by the launch script.' + default=['all'], + help='Specifies which mistral server to start by the launch script. ' + 'Valid options are all or any combination of ' + 'api, engine, and executor.' )