32e8c3da61
* Implemented Worker to be started on remote host for handling tasks request. * Implemented WorkerTaskExecutor that proxies tasks requests to remote workers. * Implemented Proxy that is used for consuming and publishing messages by Worker and Executor. * Added worker-based engine and worker task executor. * Added kombu dependency to requirements. * Added worker-based flow example. * Added unit-tests for worker-based flow components. Implements: blueprint worker-based-engine Change-Id: I8c6859ba4a1a56c2592e3d67cdfb8968b13ee99c
127 lines
4.8 KiB
Python
127 lines
4.8 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
|
|
|
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import kombu
|
|
import logging
|
|
import socket
|
|
import threading
|
|
|
|
from amqp import exceptions as amqp_exc
|
|
|
|
from taskflow.engines.worker_based import protocol as pr
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
# NOTE(skudriashev): A timeout of 1 is often used in environments where
|
|
# the socket can get "stuck", and is a best practice for Kombu consumers.
|
|
DRAIN_EVENTS_PERIOD = 1
|
|
|
|
|
|
class Proxy(object):
|
|
"""Proxy picks up messages from the named exchange, calls on_message
|
|
callback when new message received and is used to publish messages.
|
|
"""
|
|
|
|
def __init__(self, uuid, exchange_name, on_message, on_wait=None,
|
|
**kwargs):
|
|
self._uuid = uuid
|
|
self._exchange_name = exchange_name
|
|
self._on_message = on_message
|
|
self._on_wait = on_wait
|
|
self._running = threading.Event()
|
|
self._url = kwargs.get('url')
|
|
self._transport = kwargs.get('transport')
|
|
self._transport_opts = kwargs.get('transport_options')
|
|
|
|
# create connection
|
|
self._conn = kombu.Connection(self._url, transport=self._transport,
|
|
transport_options=self._transport_opts)
|
|
|
|
# create exchange
|
|
self._exchange = kombu.Exchange(name=self._exchange_name,
|
|
channel=self._conn,
|
|
durable=False,
|
|
auto_delete=True)
|
|
|
|
@property
|
|
def is_running(self):
|
|
"""Return whether proxy is running."""
|
|
return self._running.is_set()
|
|
|
|
def _make_queue(self, name, exchange, **kwargs):
|
|
"""Make named queue for the given exchange."""
|
|
queue_arguments = {'x-expires': pr.QUEUE_EXPIRE_TIMEOUT * 1000}
|
|
return kombu.Queue(name="%s_%s" % (self._exchange_name, name),
|
|
exchange=exchange,
|
|
routing_key=name,
|
|
durable=False,
|
|
queue_arguments=queue_arguments,
|
|
**kwargs)
|
|
|
|
def publish(self, msg, task_uuid, routing_key, **kwargs):
|
|
"""Publish message to the named exchange with routing key."""
|
|
with kombu.producers[self._conn].acquire(block=True) as producer:
|
|
queue = self._make_queue(routing_key, self._exchange)
|
|
producer.publish(body=msg,
|
|
routing_key=routing_key,
|
|
exchange=self._exchange,
|
|
correlation_id=task_uuid,
|
|
declare=[queue],
|
|
**kwargs)
|
|
|
|
def start(self):
|
|
"""Start proxy."""
|
|
LOG.info("Starting to consume from the '%s' exchange." %
|
|
self._exchange_name)
|
|
with kombu.connections[self._conn].acquire(block=True) as conn:
|
|
queue = self._make_queue(self._uuid, self._exchange, channel=conn)
|
|
try:
|
|
with conn.Consumer(queues=queue,
|
|
callbacks=[self._on_message]):
|
|
self._running.set()
|
|
while self.is_running:
|
|
try:
|
|
conn.drain_events(timeout=DRAIN_EVENTS_PERIOD)
|
|
except socket.timeout:
|
|
pass
|
|
if self._on_wait is not None:
|
|
self._on_wait()
|
|
finally:
|
|
try:
|
|
queue.delete(if_unused=True)
|
|
except (amqp_exc.PreconditionFailed, amqp_exc.NotFound):
|
|
pass
|
|
except Exception as e:
|
|
LOG.error("Failed to delete the '%s' queue: %s" %
|
|
(queue.name, e))
|
|
try:
|
|
self._exchange.delete(if_unused=True)
|
|
except (amqp_exc.PreconditionFailed, amqp_exc.NotFound):
|
|
pass
|
|
except Exception as e:
|
|
LOG.error("Failed to delete the '%s' exchange: %s" %
|
|
(self._exchange.name, e))
|
|
|
|
def wait(self):
|
|
"""Wait until proxy is started."""
|
|
self._running.wait()
|
|
|
|
def stop(self):
|
|
"""Stop proxy."""
|
|
self._running.clear()
|