diff --git a/doc/source/AMQP1.0.rst b/doc/source/AMQP1.0.rst index 5dab29bfe..1483d321c 100644 --- a/doc/source/AMQP1.0.rst +++ b/doc/source/AMQP1.0.rst @@ -2,7 +2,7 @@ AMQP 1.0 Protocol Support ------------------------- -.. currentmodule:: oslo.messaging +.. currentmodule:: oslo_messaging ============ Introduction diff --git a/doc/source/conffixture.rst b/doc/source/conffixture.rst index 792fadc0d..e66887d81 100644 --- a/doc/source/conffixture.rst +++ b/doc/source/conffixture.rst @@ -2,7 +2,7 @@ Testing Configurations ---------------------- -.. currentmodule:: oslo.messaging.conffixture +.. currentmodule:: oslo_messaging.conffixture .. autoclass:: ConfFixture :members: diff --git a/doc/source/exceptions.rst b/doc/source/exceptions.rst index 02dd8de0e..2f0170598 100644 --- a/doc/source/exceptions.rst +++ b/doc/source/exceptions.rst @@ -2,7 +2,7 @@ Exceptions ---------- -.. currentmodule:: oslo.messaging +.. currentmodule:: oslo_messaging .. autoexception:: ClientSendError .. autoexception:: DriverLoadFailure diff --git a/doc/source/executors.rst b/doc/source/executors.rst index 892290535..75205322c 100644 --- a/doc/source/executors.rst +++ b/doc/source/executors.rst @@ -2,9 +2,9 @@ Executors --------- -.. automodule:: oslo.messaging._executors +.. automodule:: oslo_messaging._executors -.. currentmodule:: oslo.messaging +.. currentmodule:: oslo_messaging ============== Executor types diff --git a/doc/source/notification_listener.rst b/doc/source/notification_listener.rst index 4fa06617d..0f555e5f0 100644 --- a/doc/source/notification_listener.rst +++ b/doc/source/notification_listener.rst @@ -2,9 +2,9 @@ Notification Listener --------------------- -.. automodule:: oslo.messaging.notify.listener +.. automodule:: oslo_messaging.notify.listener -.. currentmodule:: oslo.messaging +.. currentmodule:: oslo_messaging .. autofunction:: get_notification_listener diff --git a/doc/source/notifier.rst b/doc/source/notifier.rst index cfd321850..202011bea 100644 --- a/doc/source/notifier.rst +++ b/doc/source/notifier.rst @@ -2,7 +2,7 @@ Notifier -------- -.. currentmodule:: oslo.messaging +.. currentmodule:: oslo_messaging .. autoclass:: Notifier :members: diff --git a/doc/source/opts.rst b/doc/source/opts.rst index 663bbb9f8..8fc2bc736 100644 --- a/doc/source/opts.rst +++ b/doc/source/opts.rst @@ -2,6 +2,6 @@ Configuration Options ---------------------- -.. currentmodule:: oslo.messaging.opts +.. currentmodule:: oslo_messaging.opts .. autofunction:: list_opts diff --git a/doc/source/rpcclient.rst b/doc/source/rpcclient.rst index 5baada56e..60da0cc0c 100644 --- a/doc/source/rpcclient.rst +++ b/doc/source/rpcclient.rst @@ -2,7 +2,7 @@ RPC Client ---------- -.. currentmodule:: oslo.messaging +.. currentmodule:: oslo_messaging .. autoclass:: RPCClient :members: diff --git a/doc/source/serializer.rst b/doc/source/serializer.rst index 5b02bbd18..64b5d4563 100644 --- a/doc/source/serializer.rst +++ b/doc/source/serializer.rst @@ -2,7 +2,7 @@ Serializer ---------- -.. currentmodule:: oslo.messaging +.. currentmodule:: oslo_messaging .. autoclass:: Serializer :members: diff --git a/doc/source/server.rst b/doc/source/server.rst index b7ac48650..36caa0445 100644 --- a/doc/source/server.rst +++ b/doc/source/server.rst @@ -2,9 +2,9 @@ Server ------ -.. automodule:: oslo.messaging.rpc.server +.. automodule:: oslo_messaging.rpc.server -.. currentmodule:: oslo.messaging +.. currentmodule:: oslo_messaging .. autofunction:: get_rpc_server diff --git a/doc/source/target.rst b/doc/source/target.rst index 047a8e357..57a627c2c 100644 --- a/doc/source/target.rst +++ b/doc/source/target.rst @@ -2,7 +2,7 @@ Target ------ -.. currentmodule:: oslo.messaging +.. currentmodule:: oslo_messaging .. autoclass:: Target diff --git a/doc/source/transport.rst b/doc/source/transport.rst index b3076c69c..547198aa8 100644 --- a/doc/source/transport.rst +++ b/doc/source/transport.rst @@ -2,7 +2,7 @@ Transport --------- -.. currentmodule:: oslo.messaging +.. currentmodule:: oslo_messaging .. autofunction:: get_transport diff --git a/oslo/messaging/__init__.py b/oslo/messaging/__init__.py index 453a73ea2..125c96a75 100644 --- a/oslo/messaging/__init__.py +++ b/oslo/messaging/__init__.py @@ -13,6 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. +import warnings + from .exceptions import * from .localcontext import * from .notify import * @@ -21,3 +23,16 @@ from .serializer import * from .server import * from .target import * from .transport import * + + +def deprecated(): + new_name = __name__.replace('.', '_') + warnings.warn( + ('The oslo namespace package is deprecated. Please use %s instead.' % + new_name), + DeprecationWarning, + stacklevel=3, + ) + + +deprecated() diff --git a/oslo/messaging/conffixture.py b/oslo/messaging/conffixture.py index 97f6bf19d..8b4be93a8 100644 --- a/oslo/messaging/conffixture.py +++ b/oslo/messaging/conffixture.py @@ -1,6 +1,3 @@ - -# Copyright 2013 Red Hat, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -13,66 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -__all__ = ['ConfFixture'] - -import sys - -import fixtures - - -def _import_opts(conf, module, opts): - __import__(module) - conf.register_opts(getattr(sys.modules[module], opts)) - - -class ConfFixture(fixtures.Fixture): - - """Tweak configuration options for unit testing. - - oslo.messaging registers a number of configuration options, but rather than - directly referencing those options, users of the API should use this - interface for querying and overriding certain configuration options. - - An example usage:: - - self.messaging_conf = self.useFixture(messaging.ConfFixture(cfg.CONF)) - self.messaging_conf.transport_driver = 'fake' - - :param conf: a ConfigOpts instance - :type conf: oslo.config.cfg.ConfigOpts - """ - - def __init__(self, conf): - self.conf = conf - _import_opts(self.conf, - 'oslo.messaging._drivers.impl_rabbit', 'rabbit_opts') - _import_opts(self.conf, - 'oslo.messaging._drivers.impl_qpid', 'qpid_opts') - _import_opts(self.conf, - 'oslo.messaging._drivers.amqp', 'amqp_opts') - _import_opts(self.conf, 'oslo.messaging.rpc.client', '_client_opts') - _import_opts(self.conf, 'oslo.messaging.transport', '_transport_opts') - _import_opts(self.conf, - 'oslo.messaging.notify.notifier', '_notifier_opts') - - def setUp(self): - super(ConfFixture, self).setUp() - self.addCleanup(self.conf.reset) - - @property - def transport_driver(self): - """The transport driver - for example 'rabbit', 'qpid' or 'fake'.""" - return self.conf.rpc_backend - - @transport_driver.setter - def transport_driver(self, value): - self.conf.set_override('rpc_backend', value) - - @property - def response_timeout(self): - """Default number of seconds to wait for a response from a call.""" - return self.conf.rpc_response_timeout - - @response_timeout.setter - def response_timeout(self, value): - self.conf.set_override('rpc_response_timeout', value) +from oslo_messaging.conffixture import * # noqa diff --git a/oslo/messaging/exceptions.py b/oslo/messaging/exceptions.py index 93f525abe..4708d87c7 100644 --- a/oslo/messaging/exceptions.py +++ b/oslo/messaging/exceptions.py @@ -1,6 +1,3 @@ - -# Copyright 2013 Red Hat, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -13,28 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -__all__ = ['MessagingException', 'MessagingTimeout', 'MessageDeliveryFailure', - 'InvalidTarget'] - -import six - - -class MessagingException(Exception): - """Base class for exceptions.""" - - -class MessagingTimeout(MessagingException): - """Raised if message sending times out.""" - - -class MessageDeliveryFailure(MessagingException): - """Raised if message sending failed after the asked retry.""" - - -class InvalidTarget(MessagingException, ValueError): - """Raised if a target does not meet certain pre-conditions.""" - - def __init__(self, msg, target): - msg = msg + ":" + six.text_type(target) - super(InvalidTarget, self).__init__(msg) - self.target = target +from oslo_messaging.exceptions import * # noqa diff --git a/oslo/messaging/localcontext.py b/oslo/messaging/localcontext.py index 8331152d5..0b24f7f23 100644 --- a/oslo/messaging/localcontext.py +++ b/oslo/messaging/localcontext.py @@ -1,6 +1,3 @@ - -# Copyright 2013 Red Hat, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -13,43 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -__all__ = [ - 'get_local_context', - 'set_local_context', - 'clear_local_context', -] - -import threading -import uuid - -_KEY = '_%s_%s' % (__name__.replace('.', '_'), uuid.uuid4().hex) -_STORE = threading.local() - - -def get_local_context(ctxt): - """Retrieve the RPC endpoint request context for the current thread. - - This method allows any code running in the context of a dispatched RPC - endpoint method to retrieve the context for this request. - - This is commonly used for logging so that, for example, you can include the - request ID, user and tenant in every message logged from a RPC endpoint - method. - - :returns: the context for the request dispatched in the current thread - """ - return getattr(_STORE, _KEY, None) - - -def set_local_context(ctxt): - """Set the request context for the current thread. - - :param ctxt: a deserialized request context - :type ctxt: dict - """ - setattr(_STORE, _KEY, ctxt) - - -def clear_local_context(): - """Clear the request context for the current thread.""" - delattr(_STORE, _KEY) +from oslo_messaging.localcontext import * # noqa diff --git a/oslo/messaging/notify/dispatcher.py b/oslo/messaging/notify/dispatcher.py index de04a8708..d472674ad 100644 --- a/oslo/messaging/notify/dispatcher.py +++ b/oslo/messaging/notify/dispatcher.py @@ -1,7 +1,3 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# Copyright 2013 eNovance -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -14,122 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -import contextlib -import itertools -import logging -import sys - -from oslo.messaging import localcontext -from oslo.messaging import serializer as msg_serializer - - -LOG = logging.getLogger(__name__) - -PRIORITIES = ['audit', 'debug', 'info', 'warn', 'error', 'critical', 'sample'] - - -class NotificationResult(object): - HANDLED = 'handled' - REQUEUE = 'requeue' - - -class NotificationDispatcher(object): - """A message dispatcher which understands Notification messages. - - A MessageHandlingServer is constructed by passing a callable dispatcher - which is invoked with context and message dictionaries each time a message - is received. - - NotifcationDispatcher is one such dispatcher which pass a raw notification - message to the endpoints - """ - - def __init__(self, targets, endpoints, serializer, allow_requeue, - pool=None): - self.targets = targets - self.endpoints = endpoints - self.serializer = serializer or msg_serializer.NoOpSerializer() - self.allow_requeue = allow_requeue - self.pool = pool - - self._callbacks_by_priority = {} - for endpoint, prio in itertools.product(endpoints, PRIORITIES): - if hasattr(endpoint, prio): - method = getattr(endpoint, prio) - self._callbacks_by_priority.setdefault(prio, []).append(method) - - priorities = self._callbacks_by_priority.keys() - self._targets_priorities = set(itertools.product(self.targets, - priorities)) - - def _listen(self, transport): - return transport._listen_for_notifications(self._targets_priorities, - pool=self.pool) - - @contextlib.contextmanager - def __call__(self, incoming, executor_callback=None): - result_wrapper = [] - - yield lambda: result_wrapper.append( - self._dispatch_and_handle_error(incoming, executor_callback)) - - if result_wrapper[0] == NotificationResult.HANDLED: - incoming.acknowledge() - else: - incoming.requeue() - - def _dispatch_and_handle_error(self, incoming, executor_callback): - """Dispatch a notification message to the appropriate endpoint method. - - :param incoming: the incoming notification message - :type ctxt: IncomingMessage - """ - try: - return self._dispatch(incoming.ctxt, incoming.message, - executor_callback) - except Exception: - # sys.exc_info() is deleted by LOG.exception(). - exc_info = sys.exc_info() - LOG.error('Exception during message handling', - exc_info=exc_info) - return NotificationResult.HANDLED - - def _dispatch(self, ctxt, message, executor_callback=None): - """Dispatch an RPC message to the appropriate endpoint method. - - :param ctxt: the request context - :type ctxt: dict - :param message: the message payload - :type message: dict - """ - ctxt = self.serializer.deserialize_context(ctxt) - - publisher_id = message.get('publisher_id') - event_type = message.get('event_type') - metadata = { - 'message_id': message.get('message_id'), - 'timestamp': message.get('timestamp') - } - priority = message.get('priority', '').lower() - if priority not in PRIORITIES: - LOG.warning('Unknown priority "%s"', priority) - return - - payload = self.serializer.deserialize_entity(ctxt, - message.get('payload')) - - for callback in self._callbacks_by_priority.get(priority, []): - localcontext.set_local_context(ctxt) - try: - if executor_callback: - ret = executor_callback(callback, ctxt, publisher_id, - event_type, payload, metadata) - else: - ret = callback(ctxt, publisher_id, event_type, payload, - metadata) - ret = NotificationResult.HANDLED if ret is None else ret - if self.allow_requeue and ret == NotificationResult.REQUEUE: - return ret - finally: - localcontext.clear_local_context() - return NotificationResult.HANDLED +from oslo_messaging.notify.dispatcher import * # noqa diff --git a/oslo/messaging/notify/listener.py b/oslo/messaging/notify/listener.py index 9548d98e2..0e73924d9 100644 --- a/oslo/messaging/notify/listener.py +++ b/oslo/messaging/notify/listener.py @@ -1,7 +1,3 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# Copyright 2013 eNovance -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -13,125 +9,5 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -""" -A notification listener exposes a number of endpoints, each of which -contain a set of methods. Each method corresponds to a notification priority. -To create a notification listener, you supply a transport, list of targets and -a list of endpoints. - -A transport can be obtained simply by calling the get_transport() method:: - - transport = messaging.get_transport(conf) - -which will load the appropriate transport driver according to the user's -messaging configuration configuration. See get_transport() for more details. - -The target supplied when creating a notification listener expresses the topic -and - optionally - the exchange to listen on. See Target for more details -on these attributes. - -Notification listener have start(), stop() and wait() messages to begin -handling requests, stop handling requests and wait for all in-process -requests to complete. - -Each notification listener is associated with an executor which integrates the -listener with a specific I/O handling framework. Currently, there are blocking -and eventlet executors available. - -A simple example of a notification listener with multiple endpoints might be:: - - from oslo.config import cfg - from oslo import messaging - - class NotificationEndpoint(object): - def warn(self, ctxt, publisher_id, event_type, payload, metadata): - do_something(payload) - - class ErrorEndpoint(object): - def error(self, ctxt, publisher_id, event_type, payload, metadata): - do_something(payload) - - transport = messaging.get_transport(cfg.CONF) - targets = [ - messaging.Target(topic='notifications') - messaging.Target(topic='notifications_bis') - ] - endpoints = [ - NotificationEndpoint(), - ErrorEndpoint(), - ] - pool = "listener-workers" - server = messaging.get_notification_listener(transport, targets, endpoints, - pool) - server.start() - server.wait() - -A notifier sends a notification on a topic with a priority, the notification -listener will receive this notification if the topic of this one have been set -in one of the targets and if an endpoint implements the method named like the -priority - -Parameters to endpoint methods are the request context supplied by the client, -the publisher_id of the notification message, the event_type, the payload and -metadata. The metadata parameter is a mapping containing a unique message_id -and a timestamp. - -By supplying a serializer object, a listener can deserialize a request context -and arguments from - and serialize return values to - primitive types. - -By supplying a pool name you can create multiple groups of listeners consuming -notifications and that each group only receives one copy of each -notification. - -An endpoint method can explicitly return messaging.NotificationResult.HANDLED -to acknowledge a message or messaging.NotificationResult.REQUEUE to requeue the -message. - -The message is acknowledged only if all endpoints either return -messaging.NotificationResult.HANDLED or None. - -Note that not all transport drivers implement support for requeueing. In order -to use this feature, applications should assert that the feature is available -by passing allow_requeue=True to get_notification_listener(). If the driver -does not support requeueing, it will raise NotImplementedError at this point. -""" - -from oslo.messaging.notify import dispatcher as notify_dispatcher -from oslo.messaging import server as msg_server - - -def get_notification_listener(transport, targets, endpoints, - executor='blocking', serializer=None, - allow_requeue=False, pool=None): - """Construct a notification listener - - The executor parameter controls how incoming messages will be received and - dispatched. By default, the most simple executor is used - the blocking - executor. - - If the eventlet executor is used, the threading and time library need to be - monkeypatched. - - :param transport: the messaging transport - :type transport: Transport - :param targets: the exchanges and topics to listen on - :type targets: list of Target - :param endpoints: a list of endpoint objects - :type endpoints: list - :param executor: name of a message executor - for example - 'eventlet', 'blocking' - :type executor: str - :param serializer: an optional entity serializer - :type serializer: Serializer - :param allow_requeue: whether NotificationResult.REQUEUE support is needed - :type allow_requeue: bool - :param pool: the pool name - :type pool: str - :raises: NotImplementedError - """ - transport._require_driver_features(requeue=allow_requeue) - dispatcher = notify_dispatcher.NotificationDispatcher(targets, endpoints, - serializer, - allow_requeue, pool) - return msg_server.MessageHandlingServer(transport, dispatcher, executor) +from oslo_messaging.notify.listener import * # noqa diff --git a/oslo/messaging/notify/log_handler.py b/oslo/messaging/notify/log_handler.py index 0889a1301..3ee75a082 100644 --- a/oslo/messaging/notify/log_handler.py +++ b/oslo/messaging/notify/log_handler.py @@ -1,41 +1,13 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. -import logging - -from oslo.config import cfg - - -class LoggingErrorNotificationHandler(logging.Handler): - def __init__(self, *args, **kwargs): - # NOTE(dhellmann): Avoid a cyclical import by doing this one - # at runtime. - from oslo import messaging - logging.Handler.__init__(self, *args, **kwargs) - self._transport = messaging.get_transport(cfg.CONF) - self._notifier = messaging.Notifier(self._transport, - publisher_id='error.publisher') - - def emit(self, record): - # NOTE(bnemec): Notifier registers this opt with the transport. - if ('log' in self._transport.conf.notification_driver): - # NOTE(lbragstad): If we detect that log is one of the - # notification drivers, then return. This protects from infinite - # recursion where something bad happens, it gets logged, the log - # handler sends a notification, and the log_notifier sees the - # notification and logs it. - return - self._notifier.error(None, 'error_notification', - dict(error=record.msg)) - - -PublishErrorsHandler = LoggingErrorNotificationHandler +from oslo_messaging.notify.log_handler import * # noqa diff --git a/oslo/messaging/notify/logger.py b/oslo/messaging/notify/logger.py index b1e1e771c..f32a424fa 100644 --- a/oslo/messaging/notify/logger.py +++ b/oslo/messaging/notify/logger.py @@ -1,5 +1,3 @@ -# Copyright 2013 eNovance -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -11,71 +9,5 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -""" -Driver for the Python logging package that sends log records as a notification. -""" -import logging -from oslo.config import cfg -from oslo.messaging.notify import notifier -from oslo.messaging import transport - - -class LoggingNotificationHandler(logging.Handler): - """Handler for logging to the messaging notification system. - - Each time the application logs a message using the :py:mod:`logging` - module, it will be sent as a notification. The severity used for the - notification will be the same as the one used for the log record. - - This can be used into a Python logging configuration this way:: - - [handler_notifier] - class=oslo.messaging.LoggingNotificationHandler - level=ERROR - args=('qpid:///') - - """ - - CONF = cfg.CONF - """Default configuration object used, subclass this class if you want to - use another one. - - """ - - def __init__(self, url, publisher_id=None, driver=None, - topic=None, serializer=None): - self.notifier = notifier.Notifier( - transport.get_transport(self.CONF, url), - publisher_id, driver, - topic, - serializer() if serializer else None) - logging.Handler.__init__(self) - - def emit(self, record): - """Emit the log record to the messaging notification system. - - :param record: A log record to emit. - - """ - method = getattr(self.notifier, record.levelname.lower(), None) - - if not method: - return - - method(None, - 'logrecord', - { - 'name': record.name, - 'levelno': record.levelno, - 'levelname': record.levelname, - 'exc_info': record.exc_info, - 'pathname': record.pathname, - 'lineno': record.lineno, - 'msg': record.getMessage(), - 'funcName': record.funcName, - 'thread': record.thread, - 'processName': record.processName, - 'process': record.process, - 'extra': getattr(record, 'extra', None), - }) +from oslo_messaging.notify.logger import * # noqa diff --git a/oslo/messaging/notify/middleware.py b/oslo/messaging/notify/middleware.py index 76e95adab..992b65bea 100644 --- a/oslo/messaging/notify/middleware.py +++ b/oslo/messaging/notify/middleware.py @@ -1,5 +1,3 @@ -# Copyright (c) 2013-2014 eNovance -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -12,117 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -""" -Send notifications on request - -""" -import logging -import os.path -import sys -import traceback as tb - -import six -import webob.dec - -from oslo.config import cfg -from oslo import messaging -from oslo.messaging._i18n import _LE -from oslo.messaging import notify -from oslo.messaging.openstack.common import context -from oslo.middleware import base - -LOG = logging.getLogger(__name__) - - -def log_and_ignore_error(fn): - def wrapped(*args, **kwargs): - try: - return fn(*args, **kwargs) - except Exception as e: - LOG.exception(_LE('An exception occurred processing ' - 'the API call: %s ') % e) - return wrapped - - -class RequestNotifier(base.Middleware): - """Send notification on request.""" - - @classmethod - def factory(cls, global_conf, **local_conf): - """Factory method for paste.deploy.""" - conf = global_conf.copy() - conf.update(local_conf) - - def _factory(app): - return cls(app, **conf) - return _factory - - def __init__(self, app, **conf): - self.notifier = notify.Notifier( - messaging.get_transport(cfg.CONF, conf.get('url')), - publisher_id=conf.get('publisher_id', - os.path.basename(sys.argv[0]))) - self.service_name = conf.get('service_name') - self.ignore_req_list = [x.upper().strip() for x in - conf.get('ignore_req_list', '').split(',')] - super(RequestNotifier, self).__init__(app) - - @staticmethod - def environ_to_dict(environ): - """Following PEP 333, server variables are lower case, so don't - include them. - - """ - return dict((k, v) for k, v in six.iteritems(environ) - if k.isupper() and k != 'HTTP_X_AUTH_TOKEN') - - @log_and_ignore_error - def process_request(self, request): - request.environ['HTTP_X_SERVICE_NAME'] = \ - self.service_name or request.host - payload = { - 'request': self.environ_to_dict(request.environ), - } - - self.notifier.info(context.get_admin_context(), - 'http.request', - payload) - - @log_and_ignore_error - def process_response(self, request, response, - exception=None, traceback=None): - payload = { - 'request': self.environ_to_dict(request.environ), - } - - if response: - payload['response'] = { - 'status': response.status, - 'headers': response.headers, - } - - if exception: - payload['exception'] = { - 'value': repr(exception), - 'traceback': tb.format_tb(traceback) - } - - self.notifier.info(context.get_admin_context(), - 'http.response', - payload) - - @webob.dec.wsgify - def __call__(self, req): - if req.method in self.ignore_req_list: - return req.get_response(self.application) - else: - self.process_request(req) - try: - response = req.get_response(self.application) - except Exception: - exc_type, value, traceback = sys.exc_info() - self.process_response(req, None, value, traceback) - raise - else: - self.process_response(req, response) - return response +from oslo_messaging.notify.middleware import * # noqa diff --git a/oslo/messaging/notify/notifier.py b/oslo/messaging/notify/notifier.py index 624e73fa2..0d23eb039 100644 --- a/oslo/messaging/notify/notifier.py +++ b/oslo/messaging/notify/notifier.py @@ -1,8 +1,3 @@ - -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# Copyright 2013 Red Hat, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -15,301 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -import abc -import logging -import uuid - -import six -from stevedore import named - -from oslo.config import cfg -from oslo.messaging import serializer as msg_serializer -from oslo.utils import timeutils - -_notifier_opts = [ - cfg.MultiStrOpt('notification_driver', - default=[], - help='Driver or drivers to handle sending notifications.'), - cfg.ListOpt('notification_topics', - default=['notifications', ], - deprecated_name='topics', - deprecated_group='rpc_notifier2', - help='AMQP topic used for OpenStack notifications.'), -] - -_LOG = logging.getLogger(__name__) - - -@six.add_metaclass(abc.ABCMeta) -class _Driver(object): - - def __init__(self, conf, topics, transport): - self.conf = conf - self.topics = topics - self.transport = transport - - @abc.abstractmethod - def notify(self, ctxt, msg, priority, retry): - pass - - -class Notifier(object): - - """Send notification messages. - - The Notifier class is used for sending notification messages over a - messaging transport or other means. - - Notification messages follow the following format:: - - {'message_id': six.text_type(uuid.uuid4()), - 'publisher_id': 'compute.host1', - 'timestamp': timeutils.utcnow(), - 'priority': 'WARN', - 'event_type': 'compute.create_instance', - 'payload': {'instance_id': 12, ... }} - - A Notifier object can be instantiated with a transport object and a - publisher ID: - - notifier = messaging.Notifier(get_transport(CONF), 'compute') - - and notifications are sent via drivers chosen with the notification_driver - config option and on the topics chosen with the notification_topics config - option. - - Alternatively, a Notifier object can be instantiated with a specific - driver or topic:: - - notifier = notifier.Notifier(RPC_TRANSPORT, - 'compute.host', - driver='messaging', - topic='notifications') - - Notifier objects are relatively expensive to instantiate (mostly the cost - of loading notification drivers), so it is possible to specialize a given - Notifier object with a different publisher id using the prepare() method:: - - notifier = notifier.prepare(publisher_id='compute') - notifier.info(ctxt, event_type, payload) - """ - - def __init__(self, transport, publisher_id=None, - driver=None, topic=None, - serializer=None, retry=None): - """Construct a Notifier object. - - :param transport: the transport to use for sending messages - :type transport: oslo.messaging.Transport - :param publisher_id: field in notifications sent, for example - 'compute.host1' - :type publisher_id: str - :param driver: a driver to lookup from oslo.messaging.notify.drivers - :type driver: str - :param topic: the topic which to send messages on - :type topic: str - :param serializer: an optional entity serializer - :type serializer: Serializer - :param retry: an connection retries configuration - None or -1 means to retry forever - 0 means no retry - N means N retries - :type retry: int - """ - transport.conf.register_opts(_notifier_opts) - - self.transport = transport - self.publisher_id = publisher_id - self.retry = retry - - self._driver_names = ([driver] if driver is not None - else transport.conf.notification_driver) - - self._topics = ([topic] if topic is not None - else transport.conf.notification_topics) - self._serializer = serializer or msg_serializer.NoOpSerializer() - - self._driver_mgr = named.NamedExtensionManager( - 'oslo.messaging.notify.drivers', - names=self._driver_names, - invoke_on_load=True, - invoke_args=[transport.conf], - invoke_kwds={ - 'topics': self._topics, - 'transport': self.transport, - } - ) - - _marker = object() - - def prepare(self, publisher_id=_marker, retry=_marker): - """Return a specialized Notifier instance. - - Returns a new Notifier instance with the supplied publisher_id. Allows - sending notifications from multiple publisher_ids without the overhead - of notification driver loading. - - :param publisher_id: field in notifications sent, for example - 'compute.host1' - :type publisher_id: str - :param retry: an connection retries configuration - None or -1 means to retry forever - 0 means no retry - N means N retries - :type retry: int - """ - return _SubNotifier._prepare(self, publisher_id, retry=retry) - - def _notify(self, ctxt, event_type, payload, priority, publisher_id=None, - retry=None): - payload = self._serializer.serialize_entity(ctxt, payload) - ctxt = self._serializer.serialize_context(ctxt) - - msg = dict(message_id=six.text_type(uuid.uuid4()), - publisher_id=publisher_id or self.publisher_id, - event_type=event_type, - priority=priority, - payload=payload, - timestamp=six.text_type(timeutils.utcnow())) - - def do_notify(ext): - try: - ext.obj.notify(ctxt, msg, priority, retry or self.retry) - except Exception as e: - _LOG.exception("Problem '%(e)s' attempting to send to " - "notification system. Payload=%(payload)s", - dict(e=e, payload=payload)) - - if self._driver_mgr.extensions: - self._driver_mgr.map(do_notify) - - def audit(self, ctxt, event_type, payload): - """Send a notification at audit level. - - :param ctxt: a request context dict - :type ctxt: dict - :param event_type: describes the event, for example - 'compute.create_instance' - :type event_type: str - :param payload: the notification payload - :type payload: dict - :raises: MessageDeliveryFailure - """ - self._notify(ctxt, event_type, payload, 'AUDIT') - - def debug(self, ctxt, event_type, payload): - """Send a notification at debug level. - - :param ctxt: a request context dict - :type ctxt: dict - :param event_type: describes the event, for example - 'compute.create_instance' - :type event_type: str - :param payload: the notification payload - :type payload: dict - :raises: MessageDeliveryFailure - """ - self._notify(ctxt, event_type, payload, 'DEBUG') - - def info(self, ctxt, event_type, payload): - """Send a notification at info level. - - :param ctxt: a request context dict - :type ctxt: dict - :param event_type: describes the event, for example - 'compute.create_instance' - :type event_type: str - :param payload: the notification payload - :type payload: dict - :raises: MessageDeliveryFailure - """ - self._notify(ctxt, event_type, payload, 'INFO') - - def warn(self, ctxt, event_type, payload): - """Send a notification at warning level. - - :param ctxt: a request context dict - :type ctxt: dict - :param event_type: describes the event, for example - 'compute.create_instance' - :type event_type: str - :param payload: the notification payload - :type payload: dict - :raises: MessageDeliveryFailure - """ - self._notify(ctxt, event_type, payload, 'WARN') - - warning = warn - - def error(self, ctxt, event_type, payload): - """Send a notification at error level. - - :param ctxt: a request context dict - :type ctxt: dict - :param event_type: describes the event, for example - 'compute.create_instance' - :type event_type: str - :param payload: the notification payload - :type payload: dict - :raises: MessageDeliveryFailure - """ - self._notify(ctxt, event_type, payload, 'ERROR') - - def critical(self, ctxt, event_type, payload): - """Send a notification at critical level. - - :param ctxt: a request context dict - :type ctxt: dict - :param event_type: describes the event, for example - 'compute.create_instance' - :type event_type: str - :param payload: the notification payload - :type payload: dict - :raises: MessageDeliveryFailure - """ - self._notify(ctxt, event_type, payload, 'CRITICAL') - - def sample(self, ctxt, event_type, payload): - """Send a notification at sample level. - - Sample notifications are for high-frequency events - that typically contain small payloads. eg: "CPU = 70%" - - Not all drivers support the sample level - (log, for example) so these could be dropped. - - :param ctxt: a request context dict - :type ctxt: dict - :param event_type: describes the event, for example - 'compute.create_instance' - :type event_type: str - :param payload: the notification payload - :type payload: dict - :raises: MessageDeliveryFailure - """ - self._notify(ctxt, event_type, payload, 'SAMPLE') - - -class _SubNotifier(Notifier): - - _marker = Notifier._marker - - def __init__(self, base, publisher_id, retry): - self._base = base - self.transport = base.transport - self.publisher_id = publisher_id - self.retry = retry - - self._serializer = self._base._serializer - self._driver_mgr = self._base._driver_mgr - - def _notify(self, ctxt, event_type, payload, priority): - super(_SubNotifier, self)._notify(ctxt, event_type, payload, priority) - - @classmethod - def _prepare(cls, base, publisher_id=_marker, retry=_marker): - if publisher_id is cls._marker: - publisher_id = base.publisher_id - if retry is cls._marker: - retry = base.retry - return cls(base, publisher_id, retry=retry) +from oslo_messaging.notify.notifier import * # noqa diff --git a/oslo/messaging/rpc/client.py b/oslo/messaging/rpc/client.py index 7bd373718..c625ba2e9 100644 --- a/oslo/messaging/rpc/client.py +++ b/oslo/messaging/rpc/client.py @@ -1,9 +1,3 @@ - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# Copyright 2013 Red Hat, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -16,382 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -__all__ = [ - 'ClientSendError', - 'RPCClient', - 'RPCVersionCapError', - 'RemoteError', -] - -import six - -from oslo.config import cfg -from oslo.messaging._drivers import base as driver_base -from oslo.messaging import _utils as utils -from oslo.messaging import exceptions -from oslo.messaging import serializer as msg_serializer - -_client_opts = [ - cfg.IntOpt('rpc_response_timeout', - default=60, - help='Seconds to wait for a response from a call.'), -] - - -class RemoteError(exceptions.MessagingException): - - """Signifies that a remote endpoint method has raised an exception. - - Contains a string representation of the type of the original exception, - the value of the original exception, and the traceback. These are - sent to the parent as a joined string so printing the exception - contains all of the relevant info. - """ - - def __init__(self, exc_type=None, value=None, traceback=None): - self.exc_type = exc_type - self.value = value - self.traceback = traceback - msg = ("Remote error: %(exc_type)s %(value)s\n%(traceback)s." % - dict(exc_type=self.exc_type, value=self.value, - traceback=self.traceback)) - super(RemoteError, self).__init__(msg) - - -class RPCVersionCapError(exceptions.MessagingException): - - def __init__(self, version, version_cap): - self.version = version - self.version_cap = version_cap - msg = ("Requested message version, %(version)s is too high. It needs " - "to be lower than the specified version cap %(version_cap)s." % - dict(version=self.version, version_cap=self.version_cap)) - super(RPCVersionCapError, self).__init__(msg) - - -class ClientSendError(exceptions.MessagingException): - """Raised if we failed to send a message to a target.""" - - def __init__(self, target, ex): - msg = 'Failed to send to target "%s": %s' % (target, ex) - super(ClientSendError, self).__init__(msg) - self.target = target - self.ex = ex - - -class _CallContext(object): - - _marker = object() - - def __init__(self, transport, target, serializer, - timeout=None, version_cap=None, retry=None): - self.conf = transport.conf - - self.transport = transport - self.target = target - self.serializer = serializer - self.timeout = timeout - self.retry = retry - self.version_cap = version_cap - - super(_CallContext, self).__init__() - - def _make_message(self, ctxt, method, args): - msg = dict(method=method) - - msg['args'] = dict() - for argname, arg in six.iteritems(args): - msg['args'][argname] = self.serializer.serialize_entity(ctxt, arg) - - if self.target.namespace is not None: - msg['namespace'] = self.target.namespace - if self.target.version is not None: - msg['version'] = self.target.version - - return msg - - def _check_version_cap(self, version): - if not utils.version_is_compatible(self.version_cap, version): - raise RPCVersionCapError(version=version, - version_cap=self.version_cap) - - def can_send_version(self, version=_marker): - """Check to see if a version is compatible with the version cap.""" - version = self.target.version if version is self._marker else version - return (not self.version_cap or - utils.version_is_compatible(self.version_cap, - self.target.version)) - - def cast(self, ctxt, method, **kwargs): - """Invoke a method and return immediately. See RPCClient.cast().""" - msg = self._make_message(ctxt, method, kwargs) - ctxt = self.serializer.serialize_context(ctxt) - - if self.version_cap: - self._check_version_cap(msg.get('version')) - try: - self.transport._send(self.target, ctxt, msg, retry=self.retry) - except driver_base.TransportDriverError as ex: - raise ClientSendError(self.target, ex) - - def call(self, ctxt, method, **kwargs): - """Invoke a method and wait for a reply. See RPCClient.call().""" - if self.target.fanout: - raise exceptions.InvalidTarget('A call cannot be used with fanout', - self.target) - - msg = self._make_message(ctxt, method, kwargs) - msg_ctxt = self.serializer.serialize_context(ctxt) - - timeout = self.timeout - if self.timeout is None: - timeout = self.conf.rpc_response_timeout - - if self.version_cap: - self._check_version_cap(msg.get('version')) - - try: - result = self.transport._send(self.target, msg_ctxt, msg, - wait_for_reply=True, timeout=timeout, - retry=self.retry) - except driver_base.TransportDriverError as ex: - raise ClientSendError(self.target, ex) - return self.serializer.deserialize_entity(ctxt, result) - - @classmethod - def _prepare(cls, base, - exchange=_marker, topic=_marker, namespace=_marker, - version=_marker, server=_marker, fanout=_marker, - timeout=_marker, version_cap=_marker, retry=_marker): - """Prepare a method invocation context. See RPCClient.prepare().""" - kwargs = dict( - exchange=exchange, - topic=topic, - namespace=namespace, - version=version, - server=server, - fanout=fanout) - kwargs = dict([(k, v) for k, v in kwargs.items() - if v is not cls._marker]) - target = base.target(**kwargs) - - if timeout is cls._marker: - timeout = base.timeout - if retry is cls._marker: - retry = base.retry - if version_cap is cls._marker: - version_cap = base.version_cap - - return _CallContext(base.transport, target, - base.serializer, - timeout, version_cap, retry) - - def prepare(self, exchange=_marker, topic=_marker, namespace=_marker, - version=_marker, server=_marker, fanout=_marker, - timeout=_marker, version_cap=_marker, retry=_marker): - """Prepare a method invocation context. See RPCClient.prepare().""" - return self._prepare(self, - exchange, topic, namespace, - version, server, fanout, - timeout, version_cap, retry) - - -class RPCClient(object): - - """A class for invoking methods on remote servers. - - The RPCClient class is responsible for sending method invocations to remote - servers via a messaging transport. - - A default target is supplied to the RPCClient constructor, but target - attributes can be overridden for individual method invocations using the - prepare() method. - - A method invocation consists of a request context dictionary, a method name - and a dictionary of arguments. A cast() invocation just sends the request - and returns immediately. A call() invocation waits for the server to send - a return value. - - This class is intended to be used by wrapping it in another class which - provides methods on the subclass to perform the remote invocation using - call() or cast():: - - class TestClient(object): - - def __init__(self, transport): - target = messaging.Target(topic='testtopic', version='2.0') - self._client = messaging.RPCClient(transport, target) - - def test(self, ctxt, arg): - return self._client.call(ctxt, 'test', arg=arg) - - An example of using the prepare() method to override some attributes of the - default target:: - - def test(self, ctxt, arg): - cctxt = self._client.prepare(version='2.5') - return cctxt.call(ctxt, 'test', arg=arg) - - RPCClient have a number of other properties - for example, timeout and - version_cap - which may make sense to override for some method invocations, - so they too can be passed to prepare():: - - def test(self, ctxt, arg): - cctxt = self._client.prepare(timeout=10) - return cctxt.call(ctxt, 'test', arg=arg) - - However, this class can be used directly without wrapping it another class. - For example:: - - transport = messaging.get_transport(cfg.CONF) - target = messaging.Target(topic='testtopic', version='2.0') - client = messaging.RPCClient(transport, target) - client.call(ctxt, 'test', arg=arg) - - but this is probably only useful in limited circumstances as a wrapper - class will usually help to make the code much more obvious. - - By default, cast() and call() will block until the message is successfully - sent. However, the retry parameter can be used to have message sending - fail with a MessageDeliveryFailure after the given number of retries. For - example:: - - client = messaging.RPCClient(transport, target, retry=None) - client.call(ctxt, 'sync') - try: - client.prepare(retry=0).cast(ctxt, 'ping') - except messaging.MessageDeliveryFailure: - LOG.error("Failed to send ping message") - """ - - def __init__(self, transport, target, - timeout=None, version_cap=None, serializer=None, retry=None): - """Construct an RPC client. - - :param transport: a messaging transport handle - :type transport: Transport - :param target: the default target for invocations - :type target: Target - :param timeout: an optional default timeout (in seconds) for call()s - :type timeout: int or float - :param version_cap: raise a RPCVersionCapError version exceeds this cap - :type version_cap: str - :param serializer: an optional entity serializer - :type serializer: Serializer - :param retry: an optional default connection retries configuration - None or -1 means to retry forever - 0 means no retry - N means N retries - :type retry: int - """ - self.conf = transport.conf - self.conf.register_opts(_client_opts) - - self.transport = transport - self.target = target - self.timeout = timeout - self.retry = retry - self.version_cap = version_cap - self.serializer = serializer or msg_serializer.NoOpSerializer() - - super(RPCClient, self).__init__() - - _marker = _CallContext._marker - - def prepare(self, exchange=_marker, topic=_marker, namespace=_marker, - version=_marker, server=_marker, fanout=_marker, - timeout=_marker, version_cap=_marker, retry=_marker): - """Prepare a method invocation context. - - Use this method to override client properties for an individual method - invocation. For example:: - - def test(self, ctxt, arg): - cctxt = self.prepare(version='2.5') - return cctxt.call(ctxt, 'test', arg=arg) - - :param exchange: see Target.exchange - :type exchange: str - :param topic: see Target.topic - :type topic: str - :param namespace: see Target.namespace - :type namespace: str - :param version: requirement the server must support, see Target.version - :type version: str - :param server: send to a specific server, see Target.server - :type server: str - :param fanout: send to all servers on topic, see Target.fanout - :type fanout: bool - :param timeout: an optional default timeout (in seconds) for call()s - :type timeout: int or float - :param version_cap: raise a RPCVersionCapError version exceeds this cap - :type version_cap: str - :param retry: an optional connection retries configuration - None or -1 means to retry forever - 0 means no retry - N means N retries - :type retry: int - """ - return _CallContext._prepare(self, - exchange, topic, namespace, - version, server, fanout, - timeout, version_cap, retry) - - def cast(self, ctxt, method, **kwargs): - """Invoke a method and return immediately. - - Method arguments must either be primitive types or types supported by - the client's serializer (if any). - - Similarly, the request context must be a dict unless the client's - serializer supports serializing another type. - - :param ctxt: a request context dict - :type ctxt: dict - :param method: the method name - :type method: str - :param kwargs: a dict of method arguments - :type kwargs: dict - :raises: MessageDeliveryFailure - """ - self.prepare().cast(ctxt, method, **kwargs) - - def call(self, ctxt, method, **kwargs): - """Invoke a method and wait for a reply. - - Method arguments must either be primitive types or types supported by - the client's serializer (if any). Similarly, the request context must - be a dict unless the client's serializer supports serializing another - type. - - The semantics of how any errors raised by the remote RPC endpoint - method are handled are quite subtle. - - Firstly, if the remote exception is contained in one of the modules - listed in the allow_remote_exmods messaging.get_transport() parameter, - then it this exception will be re-raised by call(). However, such - locally re-raised remote exceptions are distinguishable from the same - exception type raised locally because re-raised remote exceptions are - modified such that their class name ends with the '_Remote' suffix so - you may do:: - - if ex.__class__.__name__.endswith('_Remote'): - # Some special case for locally re-raised remote exceptions - - Secondly, if a remote exception is not from a module listed in the - allowed_remote_exmods list, then a messaging.RemoteError exception is - raised with all details of the remote exception. - - :param ctxt: a request context dict - :type ctxt: dict - :param method: the method name - :type method: str - :param kwargs: a dict of method arguments - :type kwargs: dict - :raises: MessagingTimeout, RemoteError, MessageDeliveryFailure - """ - return self.prepare().call(ctxt, method, **kwargs) - - def can_send_version(self, version=_marker): - """Check to see if a version is compatible with the version cap.""" - return self.prepare(version=version).can_send_version() +from oslo_messaging.rpc.client import * # noqa diff --git a/oslo/messaging/rpc/dispatcher.py b/oslo/messaging/rpc/dispatcher.py index 3b2941aa7..0cf387106 100644 --- a/oslo/messaging/rpc/dispatcher.py +++ b/oslo/messaging/rpc/dispatcher.py @@ -1,9 +1,3 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# Copyright 2013 Red Hat, Inc. -# Copyright 2013 New Dream Network, LLC (DreamHost) -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -16,180 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -__all__ = [ - 'NoSuchMethod', - 'RPCDispatcher', - 'RPCDispatcherError', - 'UnsupportedVersion', - 'ExpectedException', -] - -import contextlib -import logging -import sys - -import six - -from oslo.messaging._i18n import _ -from oslo.messaging import _utils as utils -from oslo.messaging import localcontext -from oslo.messaging import serializer as msg_serializer -from oslo.messaging import server as msg_server -from oslo.messaging import target as msg_target - -LOG = logging.getLogger(__name__) - - -class ExpectedException(Exception): - """Encapsulates an expected exception raised by an RPC endpoint - - Merely instantiating this exception records the current exception - information, which will be passed back to the RPC client without - exceptional logging. - """ - def __init__(self): - self.exc_info = sys.exc_info() - - -class RPCDispatcherError(msg_server.MessagingServerError): - "A base class for all RPC dispatcher exceptions." - - -class NoSuchMethod(RPCDispatcherError, AttributeError): - "Raised if there is no endpoint which exposes the requested method." - - def __init__(self, method): - msg = "Endpoint does not support RPC method %s" % method - super(NoSuchMethod, self).__init__(msg) - self.method = method - - -class UnsupportedVersion(RPCDispatcherError): - "Raised if there is no endpoint which supports the requested version." - - def __init__(self, version, method=None): - msg = "Endpoint does not support RPC version %s" % version - if method: - msg = "%s. Attempted method: %s" % (msg, method) - super(UnsupportedVersion, self).__init__(msg) - self.version = version - self.method = method - - -class RPCDispatcher(object): - """A message dispatcher which understands RPC messages. - - A MessageHandlingServer is constructed by passing a callable dispatcher - which is invoked with context and message dictionaries each time a message - is received. - - RPCDispatcher is one such dispatcher which understands the format of RPC - messages. The dispatcher looks at the namespace, version and method values - in the message and matches those against a list of available endpoints. - - Endpoints may have a target attribute describing the namespace and version - of the methods exposed by that object. All public methods on an endpoint - object are remotely invokable by clients. - - - """ - - def __init__(self, target, endpoints, serializer): - """Construct a rpc server dispatcher. - - :param target: the exchange, topic and server to listen on - :type target: Target - """ - - self.endpoints = endpoints - self.serializer = serializer or msg_serializer.NoOpSerializer() - self._default_target = msg_target.Target() - self._target = target - - def _listen(self, transport): - return transport._listen(self._target) - - @staticmethod - def _is_namespace(target, namespace): - return namespace == target.namespace - - @staticmethod - def _is_compatible(target, version): - endpoint_version = target.version or '1.0' - return utils.version_is_compatible(endpoint_version, version) - - def _do_dispatch(self, endpoint, method, ctxt, args, executor_callback): - ctxt = self.serializer.deserialize_context(ctxt) - new_args = dict() - for argname, arg in six.iteritems(args): - new_args[argname] = self.serializer.deserialize_entity(ctxt, arg) - func = getattr(endpoint, method) - if executor_callback: - result = executor_callback(func, ctxt, **new_args) - else: - result = func(ctxt, **new_args) - return self.serializer.serialize_entity(ctxt, result) - - @contextlib.contextmanager - def __call__(self, incoming, executor_callback=None): - incoming.acknowledge() - yield lambda: self._dispatch_and_reply(incoming, executor_callback) - - def _dispatch_and_reply(self, incoming, executor_callback): - try: - incoming.reply(self._dispatch(incoming.ctxt, - incoming.message, - executor_callback)) - except ExpectedException as e: - LOG.debug(u'Expected exception during message handling (%s)', - e.exc_info[1]) - incoming.reply(failure=e.exc_info, log_failure=False) - except Exception as e: - # sys.exc_info() is deleted by LOG.exception(). - exc_info = sys.exc_info() - LOG.error(_('Exception during message handling: %s'), e, - exc_info=exc_info) - incoming.reply(failure=exc_info) - # NOTE(dhellmann): Remove circular object reference - # between the current stack frame and the traceback in - # exc_info. - del exc_info - - def _dispatch(self, ctxt, message, executor_callback=None): - """Dispatch an RPC message to the appropriate endpoint method. - - :param ctxt: the request context - :type ctxt: dict - :param message: the message payload - :type message: dict - :raises: NoSuchMethod, UnsupportedVersion - """ - method = message.get('method') - args = message.get('args', {}) - namespace = message.get('namespace') - version = message.get('version', '1.0') - - found_compatible = False - for endpoint in self.endpoints: - target = getattr(endpoint, 'target', None) - if not target: - target = self._default_target - - if not (self._is_namespace(target, namespace) and - self._is_compatible(target, version)): - continue - - if hasattr(endpoint, method): - localcontext.set_local_context(ctxt) - try: - return self._do_dispatch(endpoint, method, ctxt, args, - executor_callback) - finally: - localcontext.clear_local_context() - - found_compatible = True - - if found_compatible: - raise NoSuchMethod(method) - else: - raise UnsupportedVersion(version, method=method) +from oslo_messaging.rpc.dispatcher import * # noqa diff --git a/oslo/messaging/rpc/server.py b/oslo/messaging/rpc/server.py index 0909e9cdf..c297fd14a 100644 --- a/oslo/messaging/rpc/server.py +++ b/oslo/messaging/rpc/server.py @@ -1,6 +1,3 @@ - -# Copyright 2013 Red Hat, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -13,140 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -""" -An RPC server exposes a number of endpoints, each of which contain a set of -methods which may be invoked remotely by clients over a given transport. - -To create an RPC server, you supply a transport, target and a list of -endpoints. - -A transport can be obtained simply by calling the get_transport() method:: - - transport = messaging.get_transport(conf) - -which will load the appropriate transport driver according to the user's -messaging configuration configuration. See get_transport() for more details. - -The target supplied when creating an RPC server expresses the topic, server -name and - optionally - the exchange to listen on. See Target for more details -on these attributes. - -Each endpoint object may have a target attribute which may have namespace and -version fields set. By default, we use the 'null namespace' and version 1.0. -Incoming method calls will be dispatched to the first endpoint with the -requested method, a matching namespace and a compatible version number. - -RPC servers have start(), stop() and wait() messages to begin handling -requests, stop handling requests and wait for all in-process requests to -complete. - -A simple example of an RPC server with multiple endpoints might be:: - - from oslo.config import cfg - from oslo import messaging - - class ServerControlEndpoint(object): - - target = messaging.Target(namespace='control', - version='2.0') - - def __init__(self, server): - self.server = server - - def stop(self, ctx): - if server: - self.server.stop() - - class TestEndpoint(object): - - def test(self, ctx, arg): - return arg - - transport = messaging.get_transport(cfg.CONF) - target = messaging.Target(topic='test', server='server1') - endpoints = [ - ServerControlEndpoint(None), - TestEndpoint(), - ] - server = messaging.get_rpc_server(transport, target, endpoints, - executor='blocking') - server.start() - server.wait() - -Clients can invoke methods on the server by sending the request to a topic and -it gets sent to one of the servers listening on the topic, or by sending the -request to a specific server listening on the topic, or by sending the request -to all servers listening on the topic (known as fanout). These modes are chosen -via the server and fanout attributes on Target but the mode used is transparent -to the server. - -The first parameter to method invocations is always the request context -supplied by the client. - -Parameters to the method invocation are primitive types and so must be the -return values from the methods. By supplying a serializer object, a server can -deserialize a request context and arguments from - and serialize return values -to - primitive types. -""" - -__all__ = [ - 'get_rpc_server', - 'expected_exceptions', -] - -from oslo.messaging.rpc import dispatcher as rpc_dispatcher -from oslo.messaging import server as msg_server - - -def get_rpc_server(transport, target, endpoints, - executor='blocking', serializer=None): - """Construct an RPC server. - - The executor parameter controls how incoming messages will be received and - dispatched. By default, the most simple executor is used - the blocking - executor. - - If the eventlet executor is used, the threading and time library need to be - monkeypatched. - - :param transport: the messaging transport - :type transport: Transport - :param target: the exchange, topic and server to listen on - :type target: Target - :param endpoints: a list of endpoint objects - :type endpoints: list - :param executor: name of a message executor - for example - 'eventlet', 'blocking' - :type executor: str - :param serializer: an optional entity serializer - :type serializer: Serializer - """ - dispatcher = rpc_dispatcher.RPCDispatcher(target, endpoints, serializer) - return msg_server.MessageHandlingServer(transport, dispatcher, executor) - - -def expected_exceptions(*exceptions): - """Decorator for RPC endpoint methods that raise expected exceptions. - - Marking an endpoint method with this decorator allows the declaration - of expected exceptions that the RPC server should not consider fatal, - and not log as if they were generated in a real error scenario. - - Note that this will cause listed exceptions to be wrapped in an - ExpectedException, which is used internally by the RPC sever. The RPC - client will see the original exception type. - """ - def outer(func): - def inner(*args, **kwargs): - try: - return func(*args, **kwargs) - # Take advantage of the fact that we can catch - # multiple exception types using a tuple of - # exception classes, with subclass detection - # for free. Any exception that is not in or - # derived from the args passed to us will be - # ignored and thrown as normal. - except exceptions: - raise rpc_dispatcher.ExpectedException() - return inner - return outer +from oslo_messaging.rpc.server import * # noqa diff --git a/oslo/messaging/serializer.py b/oslo/messaging/serializer.py index 894f0f4a7..b7b9b3f68 100644 --- a/oslo/messaging/serializer.py +++ b/oslo/messaging/serializer.py @@ -1,5 +1,3 @@ -# Copyright 2013 IBM Corp. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -12,65 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -__all__ = ['Serializer', 'NoOpSerializer'] - -"""Provides the definition of a message serialization handler""" - -import abc - -import six - - -@six.add_metaclass(abc.ABCMeta) -class Serializer(object): - """Generic (de-)serialization definition base class.""" - - @abc.abstractmethod - def serialize_entity(self, ctxt, entity): - """Serialize something to primitive form. - - :param ctxt: Request context, in deserialized form - :param entity: Entity to be serialized - :returns: Serialized form of entity - """ - - @abc.abstractmethod - def deserialize_entity(self, ctxt, entity): - """Deserialize something from primitive form. - - :param ctxt: Request context, in deserialized form - :param entity: Primitive to be deserialized - :returns: Deserialized form of entity - """ - - @abc.abstractmethod - def serialize_context(self, ctxt): - """Serialize a request context into a dictionary. - - :param ctxt: Request context - :returns: Serialized form of context - """ - - @abc.abstractmethod - def deserialize_context(self, ctxt): - """Deserialize a dictionary into a request context. - - :param ctxt: Request context dictionary - :returns: Deserialized form of entity - """ - - -class NoOpSerializer(Serializer): - """A serializer that does nothing.""" - - def serialize_entity(self, ctxt, entity): - return entity - - def deserialize_entity(self, ctxt, entity): - return entity - - def serialize_context(self, ctxt): - return ctxt - - def deserialize_context(self, ctxt): - return ctxt +from oslo_messaging.serializer import * # noqa diff --git a/oslo/messaging/server.py b/oslo/messaging/server.py index c5c5722a0..517f9abe5 100644 --- a/oslo/messaging/server.py +++ b/oslo/messaging/server.py @@ -1,9 +1,3 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# Copyright 2013 Red Hat, Inc. -# Copyright 2013 New Dream Network, LLC (DreamHost) -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -16,135 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -__all__ = [ - 'ExecutorLoadFailure', - 'MessageHandlingServer', - 'MessagingServerError', - 'ServerListenError', -] - -from stevedore import driver - -from oslo.messaging._drivers import base as driver_base -from oslo.messaging import exceptions - - -class MessagingServerError(exceptions.MessagingException): - """Base class for all MessageHandlingServer exceptions.""" - - -class ExecutorLoadFailure(MessagingServerError): - """Raised if an executor can't be loaded.""" - - def __init__(self, executor, ex): - msg = 'Failed to load executor "%s": %s' % (executor, ex) - super(ExecutorLoadFailure, self).__init__(msg) - self.executor = executor - self.ex = ex - - -class ServerListenError(MessagingServerError): - """Raised if we failed to listen on a target.""" - - def __init__(self, target, ex): - msg = 'Failed to listen on target "%s": %s' % (target, ex) - super(ServerListenError, self).__init__(msg) - self.target = target - self.ex = ex - - -class MessageHandlingServer(object): - """Server for handling messages. - - Connect a transport to a dispatcher that knows how to process the - message using an executor that knows how the app wants to create - new tasks. - """ - - def __init__(self, transport, dispatcher, executor='blocking'): - """Construct a message handling server. - - The dispatcher parameter is a callable which is invoked with context - and message dictionaries each time a message is received. - - The executor parameter controls how incoming messages will be received - and dispatched. By default, the most simple executor is used - the - blocking executor. - - :param transport: the messaging transport - :type transport: Transport - :param dispatcher: a callable which is invoked for each method - :type dispatcher: callable - :param executor: name of message executor - for example - 'eventlet', 'blocking' - :type executor: str - """ - self.conf = transport.conf - - self.transport = transport - self.dispatcher = dispatcher - self.executor = executor - - try: - mgr = driver.DriverManager('oslo.messaging.executors', - self.executor) - except RuntimeError as ex: - raise ExecutorLoadFailure(self.executor, ex) - else: - self._executor_cls = mgr.driver - self._executor = None - - super(MessageHandlingServer, self).__init__() - - def start(self): - """Start handling incoming messages. - - This method causes the server to begin polling the transport for - incoming messages and passing them to the dispatcher. Message - processing will continue until the stop() method is called. - - The executor controls how the server integrates with the applications - I/O handling strategy - it may choose to poll for messages in a new - process, thread or co-operatively scheduled coroutine or simply by - registering a callback with an event loop. Similarly, the executor may - choose to dispatch messages in a new thread, coroutine or simply the - current thread. - """ - if self._executor is not None: - return - try: - listener = self.dispatcher._listen(self.transport) - except driver_base.TransportDriverError as ex: - raise ServerListenError(self.target, ex) - - self._executor = self._executor_cls(self.conf, listener, - self.dispatcher) - self._executor.start() - - def stop(self): - """Stop handling incoming messages. - - Once this method returns, no new incoming messages will be handled by - the server. However, the server may still be in the process of handling - some messages, and underlying driver resources associated to this - server are still in use. See 'wait' for more details. - """ - if self._executor is not None: - self._executor.stop() - - def wait(self): - """Wait for message processing to complete. - - After calling stop(), there may still be some some existing messages - which have not been completely processed. The wait() method blocks - until all message processing has completed. - - Once it's finished, the underlying driver resources associated to this - server are released (like closing useless network connections). - """ - if self._executor is not None: - self._executor.wait() - # Close listener connection after processing all messages - self._executor.listener.cleanup() - - self._executor = None +from oslo_messaging.server import * # noqa diff --git a/oslo/messaging/target.py b/oslo/messaging/target.py index f37a2b296..2f521a17b 100644 --- a/oslo/messaging/target.py +++ b/oslo/messaging/target.py @@ -1,6 +1,3 @@ - -# Copyright 2013 Red Hat, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -13,82 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. - -class Target(object): - - """Identifies the destination of messages. - - A Target encapsulates all the information to identify where a message - should be sent or what messages a server is listening for. - - Different subsets of the information encapsulated in a Target object is - relevant to various aspects of the API: - - creating a server: - topic and server is required; exchange is optional - an endpoint's target: - namespace and version is optional - client sending a message: - topic is required, all other attributes optional - - Its attributes are: - - :param exchange: A scope for topics. Leave unspecified to default to the - control_exchange configuration option. - :type exchange: str - :param topic: A name which identifies the set of interfaces exposed by a - server. Multiple servers may listen on a topic and messages will be - dispatched to one of the servers in a round-robin fashion. - :type topic: str - :param namespace: Identifies a particular interface (i.e. set of methods) - exposed by a server. The default interface has no namespace identifier - and is referred to as the null namespace. - :type namespace: str - :param version: Interfaces have a major.minor version number associated - with them. A minor number increment indicates a backwards compatible - change and an incompatible change is indicated by a major number bump. - Servers may implement multiple major versions and clients may require - indicate that their message requires a particular minimum minor version. - :type version: str - :param server: Clients can request that a message be directed to a specific - server, rather than just one of a pool of servers listening on the topic. - :type server: str - :param fanout: Clients may request that a message be directed to all - servers listening on a topic by setting fanout to ``True``, rather than - just one of them. - :type fanout: bool - """ - - def __init__(self, exchange=None, topic=None, namespace=None, - version=None, server=None, fanout=None): - self.exchange = exchange - self.topic = topic - self.namespace = namespace - self.version = version - self.server = server - self.fanout = fanout - - def __call__(self, **kwargs): - for a in ('exchange', 'topic', 'namespace', - 'version', 'server', 'fanout'): - kwargs.setdefault(a, getattr(self, a)) - return Target(**kwargs) - - def __eq__(self, other): - return vars(self) == vars(other) - - def __ne__(self, other): - return not self == other - - def __repr__(self): - attrs = [] - for a in ['exchange', 'topic', 'namespace', - 'version', 'server', 'fanout']: - v = getattr(self, a) - if v: - attrs.append((a, v)) - values = ', '.join(['%s=%s' % i for i in attrs]) - return '' - - def __hash__(self): - return id(self) +from oslo_messaging.target import * # noqa diff --git a/oslo/messaging/transport.py b/oslo/messaging/transport.py index ba695fba8..a10dfe446 100644 --- a/oslo/messaging/transport.py +++ b/oslo/messaging/transport.py @@ -1,10 +1,3 @@ - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# Copyright 2013 Red Hat, Inc. -# Copyright (c) 2012 Rackspace Hosting -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -17,408 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -__all__ = [ - 'DriverLoadFailure', - 'InvalidTransportURL', - 'Transport', - 'TransportHost', - 'TransportURL', - 'get_transport', - 'set_transport_defaults', -] - -import six -from six.moves.urllib import parse -from stevedore import driver - -from oslo.config import cfg -from oslo.messaging import exceptions - - -_transport_opts = [ - cfg.StrOpt('transport_url', - help='A URL representing the messaging driver to use and its ' - 'full configuration. If not set, we fall back to the ' - 'rpc_backend option and driver specific configuration.'), - cfg.StrOpt('rpc_backend', - default='rabbit', - help='The messaging driver to use, defaults to rabbit. Other ' - 'drivers include qpid and zmq.'), - cfg.StrOpt('control_exchange', - default='openstack', - help='The default exchange under which topics are scoped. May ' - 'be overridden by an exchange name specified in the ' - 'transport_url option.'), -] - - -def set_transport_defaults(control_exchange): - """Set defaults for messaging transport configuration options. - - :param control_exchange: the default exchange under which topics are scoped - :type control_exchange: str - """ - cfg.set_defaults(_transport_opts, - control_exchange=control_exchange) - - -class Transport(object): - - """A messaging transport. - - This is a mostly opaque handle for an underlying messaging transport - driver. - - It has a single 'conf' property which is the cfg.ConfigOpts instance used - to construct the transport object. - """ - - def __init__(self, driver): - self.conf = driver.conf - self._driver = driver - - def _require_driver_features(self, requeue=False): - self._driver.require_features(requeue=requeue) - - def _send(self, target, ctxt, message, wait_for_reply=None, timeout=None, - retry=None): - if not target.topic: - raise exceptions.InvalidTarget('A topic is required to send', - target) - return self._driver.send(target, ctxt, message, - wait_for_reply=wait_for_reply, - timeout=timeout, retry=retry) - - def _send_notification(self, target, ctxt, message, version, retry=None): - if not target.topic: - raise exceptions.InvalidTarget('A topic is required to send', - target) - self._driver.send_notification(target, ctxt, message, version, - retry=retry) - - def _listen(self, target): - if not (target.topic and target.server): - raise exceptions.InvalidTarget('A server\'s target must have ' - 'topic and server names specified', - target) - return self._driver.listen(target) - - def _listen_for_notifications(self, targets_and_priorities, pool): - for target, priority in targets_and_priorities: - if not target.topic: - raise exceptions.InvalidTarget('A target must have ' - 'topic specified', - target) - return self._driver.listen_for_notifications( - targets_and_priorities, pool) - - def cleanup(self): - """Release all resources associated with this transport.""" - self._driver.cleanup() - - -class InvalidTransportURL(exceptions.MessagingException): - """Raised if transport URL is invalid.""" - - def __init__(self, url, msg): - super(InvalidTransportURL, self).__init__(msg) - self.url = url - - -class DriverLoadFailure(exceptions.MessagingException): - """Raised if a transport driver can't be loaded.""" - - def __init__(self, driver, ex): - msg = 'Failed to load transport driver "%s": %s' % (driver, ex) - super(DriverLoadFailure, self).__init__(msg) - self.driver = driver - self.ex = ex - - -def get_transport(conf, url=None, allowed_remote_exmods=None, aliases=None): - """A factory method for Transport objects. - - This method will construct a Transport object from transport configuration - gleaned from the user's configuration and, optionally, a transport URL. - - If a transport URL is supplied as a parameter, any transport configuration - contained in it takes precedence. If no transport URL is supplied, but - there is a transport URL supplied in the user's configuration then that - URL will take the place of the URL parameter. In both cases, any - configuration not supplied in the transport URL may be taken from - individual configuration parameters in the user's configuration. - - An example transport URL might be:: - - rabbit://me:passwd@host:5672/virtual_host - - and can either be passed as a string or a TransportURL object. - - :param conf: the user configuration - :type conf: cfg.ConfigOpts - :param url: a transport URL - :type url: str or TransportURL - :param allowed_remote_exmods: a list of modules which a client using this - transport will deserialize remote exceptions - from - :type allowed_remote_exmods: list - :param aliases: A map of transport alias to transport name - :type aliases: dict - """ - allowed_remote_exmods = allowed_remote_exmods or [] - conf.register_opts(_transport_opts) - - if not isinstance(url, TransportURL): - url = url or conf.transport_url - parsed = TransportURL.parse(conf, url, aliases) - if not parsed.transport: - raise InvalidTransportURL(url, 'No scheme specified in "%s"' % url) - url = parsed - - kwargs = dict(default_exchange=conf.control_exchange, - allowed_remote_exmods=allowed_remote_exmods) - - try: - mgr = driver.DriverManager('oslo.messaging.drivers', - url.transport.split('+')[0], - invoke_on_load=True, - invoke_args=[conf, url], - invoke_kwds=kwargs) - except RuntimeError as ex: - raise DriverLoadFailure(url.transport, ex) - - return Transport(mgr.driver) - - -class TransportHost(object): - - """A host element of a parsed transport URL.""" - - def __init__(self, hostname=None, port=None, username=None, password=None): - self.hostname = hostname - self.port = port - self.username = username - self.password = password - - def __hash__(self): - return hash((self.hostname, self.port, self.username, self.password)) - - def __eq__(self, other): - return vars(self) == vars(other) - - def __ne__(self, other): - return not self == other - - def __repr__(self): - attrs = [] - for a in ['hostname', 'port', 'username', 'password']: - v = getattr(self, a) - if v: - attrs.append((a, repr(v))) - values = ', '.join(['%s=%s' % i for i in attrs]) - return '' - - -class TransportURL(object): - - """A parsed transport URL. - - Transport URLs take the form:: - - transport://user:pass@host1:port[,hostN:portN]/virtual_host - - i.e. the scheme selects the transport driver, you may include multiple - hosts in netloc and the path part is a "virtual host" partition path. - - :param conf: a ConfigOpts instance - :type conf: oslo.config.cfg.ConfigOpts - :param transport: a transport name for example 'rabbit' or 'qpid' - :type transport: str - :param virtual_host: a virtual host path for example '/' - :type virtual_host: str - :param hosts: a list of TransportHost objects - :type hosts: list - :param aliases: A map of transport alias to transport name - :type aliases: dict - """ - - def __init__(self, conf, transport=None, virtual_host=None, hosts=None, - aliases=None): - self.conf = conf - self.conf.register_opts(_transport_opts) - self._transport = transport - self.virtual_host = virtual_host - if hosts is None: - self.hosts = [] - else: - self.hosts = hosts - if aliases is None: - self.aliases = {} - else: - self.aliases = aliases - - @property - def transport(self): - if self._transport is None: - transport = self.conf.rpc_backend - else: - transport = self._transport - return self.aliases.get(transport, transport) - - @transport.setter - def transport(self, value): - self._transport = value - - def __hash__(self): - return hash((tuple(self.hosts), self.transport, self.virtual_host)) - - def __eq__(self, other): - return (self.transport == other.transport and - self.virtual_host == other.virtual_host and - self.hosts == other.hosts) - - def __ne__(self, other): - return not self == other - - def __repr__(self): - attrs = [] - for a in ['transport', 'virtual_host', 'hosts']: - v = getattr(self, a) - if v: - attrs.append((a, repr(v))) - values = ', '.join(['%s=%s' % i for i in attrs]) - return '' - - def __str__(self): - netlocs = [] - - for host in self.hosts: - username = host.username - password = host.password - hostname = host.hostname - port = host.port - - # Starting place for the network location - netloc = '' - - # Build the username and password portion of the transport URL - if username is not None or password is not None: - if username is not None: - netloc += parse.quote(username, '') - if password is not None: - netloc += ':%s' % parse.quote(password, '') - netloc += '@' - - # Build the network location portion of the transport URL - if hostname: - if ':' in hostname: - netloc += '[%s]' % hostname - else: - netloc += hostname - if port is not None: - netloc += ':%d' % port - - netlocs.append(netloc) - - # Assemble the transport URL - url = '%s://%s/' % (self.transport, ','.join(netlocs)) - - if self.virtual_host: - url += parse.quote(self.virtual_host) - - return url - - @classmethod - def parse(cls, conf, url, aliases=None): - """Parse an url. - - Assuming a URL takes the form of:: - - transport://user:pass@host1:port[,hostN:portN]/virtual_host - - then parse the URL and return a TransportURL object. - - Netloc is parsed following the sequence bellow: - - * It is first split by ',' in order to support multiple hosts - * Username and password should be specified for each host, in - case of lack of specification they will be omitted:: - - user:pass@host1:port1,host2:port2 - - [ - {"username": "user", "password": "pass", "host": "host1:port1"}, - {"host": "host2:port2"} - ] - - :param conf: a ConfigOpts instance - :type conf: oslo.config.cfg.ConfigOpts - :param url: The URL to parse - :type url: str - :param aliases: A map of transport alias to transport name - :type aliases: dict - :returns: A TransportURL - """ - if not url: - return cls(conf, aliases=aliases) - - if not isinstance(url, six.string_types): - raise InvalidTransportURL(url, 'Wrong URL type') - - url = parse.urlparse(url) - - # Make sure there's not a query string; that could identify - # requirements we can't comply with (for example ssl), so reject it if - # it's present - if '?' in url.path or url.query: - raise InvalidTransportURL(url.geturl(), - "Cannot comply with query string in " - "transport URL") - - virtual_host = None - if url.path.startswith('/'): - virtual_host = url.path[1:] - - hosts = [] - - username = password = '' - for host in url.netloc.split(','): - if not host: - continue - - hostname = host - username = password = port = None - - if '@' in host: - username, hostname = host.split('@', 1) - if ':' in username: - username, password = username.split(':', 1) - - if not hostname: - hostname = None - elif hostname.startswith('['): - # Find the closing ']' and extract the hostname - host_end = hostname.find(']') - if host_end < 0: - # NOTE(Vek): Identical to what Python 2.7's - # urlparse.urlparse() raises in this case - raise ValueError("Invalid IPv6 URL") - - port_text = hostname[host_end:] - hostname = hostname[1:host_end] - - # Now we need the port; this is compliant with how urlparse - # parses the port data - port = None - if ':' in port_text: - port = int(port_text.split(':', 1)[1]) - elif ':' in hostname: - hostname, port = hostname.split(':', 1) - port = int(port) - - hosts.append(TransportHost(hostname=hostname, - port=port, - username=username, - password=password)) - - return cls(conf, url.scheme, virtual_host, hosts, aliases) +from oslo_messaging.transport import * # noqa diff --git a/oslo_messaging/__init__.py b/oslo_messaging/__init__.py new file mode 100644 index 000000000..453a73ea2 --- /dev/null +++ b/oslo_messaging/__init__.py @@ -0,0 +1,23 @@ + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from .exceptions import * +from .localcontext import * +from .notify import * +from .rpc import * +from .serializer import * +from .server import * +from .target import * +from .transport import * diff --git a/oslo/messaging/_cmd/__init__.py b/oslo_messaging/_cmd/__init__.py similarity index 100% rename from oslo/messaging/_cmd/__init__.py rename to oslo_messaging/_cmd/__init__.py diff --git a/oslo/messaging/_cmd/zmq_receiver.py b/oslo_messaging/_cmd/zmq_receiver.py similarity index 91% rename from oslo/messaging/_cmd/zmq_receiver.py rename to oslo_messaging/_cmd/zmq_receiver.py index 07c9b10c3..6346d6abc 100644 --- a/oslo/messaging/_cmd/zmq_receiver.py +++ b/oslo_messaging/_cmd/zmq_receiver.py @@ -22,8 +22,8 @@ import logging import sys from oslo.config import cfg -from oslo.messaging._drivers import impl_zmq -from oslo.messaging._executors import base # FIXME(markmc) +from oslo_messaging._drivers import impl_zmq +from oslo_messaging._executors import base # FIXME(markmc) CONF = cfg.CONF CONF.register_opts(impl_zmq.zmq_opts) diff --git a/oslo/messaging/_drivers/__init__.py b/oslo_messaging/_drivers/__init__.py similarity index 100% rename from oslo/messaging/_drivers/__init__.py rename to oslo_messaging/_drivers/__init__.py diff --git a/oslo/messaging/_drivers/amqp.py b/oslo_messaging/_drivers/amqp.py similarity index 98% rename from oslo/messaging/_drivers/amqp.py rename to oslo_messaging/_drivers/amqp.py index ac9321053..0648b1ef8 100644 --- a/oslo/messaging/_drivers/amqp.py +++ b/oslo_messaging/_drivers/amqp.py @@ -30,9 +30,9 @@ import uuid import six from oslo.config import cfg -from oslo.messaging._drivers import common as rpc_common -from oslo.messaging._drivers import pool from oslo.utils import strutils +from oslo_messaging._drivers import common as rpc_common +from oslo_messaging._drivers import pool amqp_opts = [ cfg.BoolOpt('amqp_durable_queues', diff --git a/oslo/messaging/_drivers/amqpdriver.py b/oslo_messaging/_drivers/amqpdriver.py similarity index 97% rename from oslo/messaging/_drivers/amqpdriver.py rename to oslo_messaging/_drivers/amqpdriver.py index 0bebe8a79..c222bce64 100644 --- a/oslo/messaging/_drivers/amqpdriver.py +++ b/oslo_messaging/_drivers/amqpdriver.py @@ -21,12 +21,12 @@ import uuid from six import moves -from oslo import messaging -from oslo.messaging._drivers import amqp as rpc_amqp -from oslo.messaging._drivers import base -from oslo.messaging._drivers import common as rpc_common -from oslo.messaging._i18n import _ -from oslo.messaging._i18n import _LI +import oslo_messaging +from oslo_messaging._drivers import amqp as rpc_amqp +from oslo_messaging._drivers import base +from oslo_messaging._drivers import common as rpc_common +from oslo_messaging._i18n import _ +from oslo_messaging._i18n import _LI LOG = logging.getLogger(__name__) @@ -143,8 +143,9 @@ class ReplyWaiters(object): try: return self._queues[msg_id].get(block=True, timeout=timeout) except moves.queue.Empty: - raise messaging.MessagingTimeout('Timed out waiting for a reply ' - 'to message ID %s' % msg_id) + raise oslo_messaging.MessagingTimeout( + 'Timed out waiting for a reply ' + 'to message ID %s' % msg_id) def check(self, msg_id): try: @@ -207,7 +208,7 @@ class ReplyWaiter(object): @staticmethod def _raise_timeout_exception(msg_id): - raise messaging.MessagingTimeout( + raise oslo_messaging.MessagingTimeout( _('Timed out waiting for a reply to message ID %s.') % msg_id) def _process_reply(self, data): diff --git a/oslo/messaging/_drivers/base.py b/oslo_messaging/_drivers/base.py similarity index 98% rename from oslo/messaging/_drivers/base.py rename to oslo_messaging/_drivers/base.py index ffaebe206..af0866c91 100644 --- a/oslo/messaging/_drivers/base.py +++ b/oslo_messaging/_drivers/base.py @@ -17,7 +17,7 @@ import abc import six -from oslo.messaging import exceptions +from oslo_messaging import exceptions class TransportDriverError(exceptions.MessagingException): diff --git a/oslo/messaging/_drivers/common.py b/oslo_messaging/_drivers/common.py similarity index 97% rename from oslo/messaging/_drivers/common.py rename to oslo_messaging/_drivers/common.py index 3d671438c..3b0247fba 100644 --- a/oslo/messaging/_drivers/common.py +++ b/oslo_messaging/_drivers/common.py @@ -23,10 +23,10 @@ import traceback import six -from oslo import messaging -from oslo.messaging._i18n import _ -from oslo.messaging import _utils as utils from oslo.serialization import jsonutils +import oslo_messaging +from oslo_messaging._i18n import _ +from oslo_messaging import _utils as utils LOG = logging.getLogger(__name__) @@ -211,7 +211,7 @@ def deserialize_remote_exception(data, allowed_remote_exmods): # NOTE(ameade): We DO NOT want to allow just any module to be imported, in # order to prevent arbitrary code execution. if module != _EXCEPTIONS_MODULE and module not in allowed_remote_exmods: - return messaging.RemoteError(name, failure.get('message'), trace) + return oslo_messaging.RemoteError(name, failure.get('message'), trace) try: __import__(module) @@ -222,7 +222,7 @@ def deserialize_remote_exception(data, allowed_remote_exmods): failure = klass(*failure.get('args', []), **failure.get('kwargs', {})) except (AttributeError, TypeError, ImportError): - return messaging.RemoteError(name, failure.get('message'), trace) + return oslo_messaging.RemoteError(name, failure.get('message'), trace) ex_type = type(failure) str_override = lambda self: message diff --git a/oslo/messaging/_drivers/impl_fake.py b/oslo_messaging/_drivers/impl_fake.py similarity index 91% rename from oslo/messaging/_drivers/impl_fake.py rename to oslo_messaging/_drivers/impl_fake.py index 1a7e87a7b..bb32d12d5 100644 --- a/oslo/messaging/_drivers/impl_fake.py +++ b/oslo_messaging/_drivers/impl_fake.py @@ -20,8 +20,8 @@ import time from six import moves -from oslo import messaging -from oslo.messaging._drivers import base +import oslo_messaging +from oslo_messaging._drivers import base class FakeIncomingMessage(base.IncomingMessage): @@ -199,7 +199,7 @@ class FakeDriver(base.BaseDriver): else: return reply except moves.queue.Empty: - raise messaging.MessagingTimeout( + raise oslo_messaging.MessagingTimeout( 'No reply on topic %s' % target.topic) return None @@ -218,17 +218,21 @@ class FakeDriver(base.BaseDriver): def listen(self, target): exchange = target.exchange or self._default_exchange listener = FakeListener(self, self._exchange_manager, - [messaging.Target(topic=target.topic, - server=target.server, - exchange=exchange), - messaging.Target(topic=target.topic, - exchange=exchange)]) + [oslo_messaging.Target( + topic=target.topic, + server=target.server, + exchange=exchange), + oslo_messaging.Target( + topic=target.topic, + exchange=exchange)]) return listener def listen_for_notifications(self, targets_and_priorities, pool): - targets = [messaging.Target(topic='%s.%s' % (target.topic, priority), - exchange=target.exchange) - for target, priority in targets_and_priorities] + targets = [ + oslo_messaging.Target( + topic='%s.%s' % (target.topic, priority), + exchange=target.exchange) + for target, priority in targets_and_priorities] listener = FakeListener(self, self._exchange_manager, targets, pool) return listener diff --git a/oslo/messaging/_drivers/impl_qpid.py b/oslo_messaging/_drivers/impl_qpid.py similarity index 98% rename from oslo/messaging/_drivers/impl_qpid.py rename to oslo_messaging/_drivers/impl_qpid.py index de477f4d4..7bd527258 100644 --- a/oslo/messaging/_drivers/impl_qpid.py +++ b/oslo_messaging/_drivers/impl_qpid.py @@ -23,14 +23,14 @@ import time import six from oslo.config import cfg -from oslo.messaging._drivers import amqp as rpc_amqp -from oslo.messaging._drivers import amqpdriver -from oslo.messaging._drivers import common as rpc_common -from oslo.messaging._i18n import _ -from oslo.messaging import exceptions from oslo.serialization import jsonutils from oslo.utils import importutils from oslo.utils import netutils +from oslo_messaging._drivers import amqp as rpc_amqp +from oslo_messaging._drivers import amqpdriver +from oslo_messaging._drivers import common as rpc_common +from oslo_messaging._i18n import _ +from oslo_messaging import exceptions qpid_codec = importutils.try_import("qpid.codec010") qpid_messaging = importutils.try_import("qpid.messaging") @@ -594,7 +594,7 @@ class Connection(object): LOG.warn("Process forked! " "This can result in unpredictable behavior. " "See: http://docs.openstack.org/developer/" - "oslo.messaging/transport.html") + "oslo_messaging/transport.html") self._initial_pid = current_pid while True: diff --git a/oslo/messaging/_drivers/impl_rabbit.py b/oslo_messaging/_drivers/impl_rabbit.py similarity index 98% rename from oslo/messaging/_drivers/impl_rabbit.py rename to oslo_messaging/_drivers/impl_rabbit.py index a93ad7bf0..f7b932e3b 100644 --- a/oslo/messaging/_drivers/impl_rabbit.py +++ b/oslo_messaging/_drivers/impl_rabbit.py @@ -30,13 +30,13 @@ import six from six.moves.urllib import parse from oslo.config import cfg -from oslo.messaging._drivers import amqp as rpc_amqp -from oslo.messaging._drivers import amqpdriver -from oslo.messaging._drivers import common as rpc_common -from oslo.messaging._i18n import _ -from oslo.messaging._i18n import _LI -from oslo.messaging import exceptions from oslo.utils import netutils +from oslo_messaging._drivers import amqp as rpc_amqp +from oslo_messaging._drivers import amqpdriver +from oslo_messaging._drivers import common as rpc_common +from oslo_messaging._i18n import _ +from oslo_messaging._i18n import _LI +from oslo_messaging import exceptions rabbit_opts = [ @@ -104,7 +104,7 @@ rabbit_opts = [ 'If you change this option, you must wipe the ' 'RabbitMQ database.'), - # NOTE(sileht): deprecated option since oslo.messaging 1.5.0, + # NOTE(sileht): deprecated option since oslo_messaging 1.5.0, cfg.BoolOpt('fake_rabbit', default=False, help='Deprecated, use rpc_backend=kombu+memory or ' @@ -583,7 +583,7 @@ class Connection(object): LOG.warn("Process forked after connection established! " "This can result in unpredictable behavior. " "See: http://docs.openstack.org/developer/" - "oslo.messaging/transport.html") + "oslo_messaging/transport.html") self._initial_pid = current_pid if retry is None: diff --git a/oslo/messaging/_drivers/impl_zmq.py b/oslo_messaging/_drivers/impl_zmq.py similarity index 98% rename from oslo/messaging/_drivers/impl_zmq.py rename to oslo_messaging/_drivers/impl_zmq.py index 868e092f1..83668d33f 100644 --- a/oslo/messaging/_drivers/impl_zmq.py +++ b/oslo_messaging/_drivers/impl_zmq.py @@ -29,10 +29,10 @@ import six from six import moves from oslo.config import cfg -from oslo.messaging._drivers import base -from oslo.messaging._drivers import common as rpc_common -from oslo.messaging._executors import base as executor_base # FIXME(markmc) -from oslo.messaging._i18n import _, _LE +from oslo_messaging._drivers import base +from oslo_messaging._drivers import common as rpc_common +from oslo_messaging._executors import base as executor_base # FIXME(markmc) +from oslo_messaging._i18n import _, _LE from oslo.serialization import jsonutils from oslo.utils import excutils from oslo.utils import importutils @@ -56,7 +56,7 @@ zmq_opts = [ # The module.Class to use for matchmaking. cfg.StrOpt( 'rpc_zmq_matchmaker', - default=('oslo.messaging._drivers.' + default=('oslo_messaging._drivers.' 'matchmaker.MatchMakerLocalhost'), help='MatchMaker driver.', ), @@ -124,7 +124,14 @@ class ZmqSocket(object): # Enable IPv6-support in libzmq. # When IPv6 is enabled, a socket will connect to, or accept # connections from, both IPv4 and IPv6 hosts. - self.sock.ipv6 = True + try: + self.sock.ipv6 = True + except AttributeError: + # NOTE(dhellmann): Sometimes the underlying library does + # not recognize the IPV6 option. There's nothing we can + # really do in that case, so ignore the error and keep + # trying to work. + pass self.addr = addr self.type = zmq_type diff --git a/oslo/messaging/_drivers/matchmaker.py b/oslo_messaging/_drivers/matchmaker.py similarity index 99% rename from oslo/messaging/_drivers/matchmaker.py rename to oslo_messaging/_drivers/matchmaker.py index 1ac332a07..0b3ba4b83 100644 --- a/oslo/messaging/_drivers/matchmaker.py +++ b/oslo_messaging/_drivers/matchmaker.py @@ -22,7 +22,7 @@ import logging import eventlet from oslo.config import cfg -from oslo.messaging._i18n import _ +from oslo_messaging._i18n import _ matchmaker_opts = [ cfg.IntOpt('matchmaker_heartbeat_freq', diff --git a/oslo/messaging/_drivers/matchmaker_redis.py b/oslo_messaging/_drivers/matchmaker_redis.py similarity index 98% rename from oslo/messaging/_drivers/matchmaker_redis.py rename to oslo_messaging/_drivers/matchmaker_redis.py index d007780ed..760561d98 100644 --- a/oslo/messaging/_drivers/matchmaker_redis.py +++ b/oslo_messaging/_drivers/matchmaker_redis.py @@ -17,8 +17,8 @@ return keys for direct exchanges, per (approximate) AMQP parlance. """ from oslo.config import cfg -from oslo.messaging._drivers import matchmaker as mm_common from oslo.utils import importutils +from oslo_messaging._drivers import matchmaker as mm_common redis = importutils.try_import('redis') diff --git a/oslo/messaging/_drivers/matchmaker_ring.py b/oslo_messaging/_drivers/matchmaker_ring.py similarity index 97% rename from oslo/messaging/_drivers/matchmaker_ring.py rename to oslo_messaging/_drivers/matchmaker_ring.py index f3f0334b0..0b65875f7 100644 --- a/oslo/messaging/_drivers/matchmaker_ring.py +++ b/oslo_messaging/_drivers/matchmaker_ring.py @@ -21,8 +21,8 @@ import json import logging from oslo.config import cfg -from oslo.messaging._drivers import matchmaker as mm -from oslo.messaging._i18n import _ +from oslo_messaging._drivers import matchmaker as mm +from oslo_messaging._i18n import _ matchmaker_opts = [ # Matchmaker ring file diff --git a/oslo/messaging/_drivers/pool.py b/oslo_messaging/_drivers/pool.py similarity index 100% rename from oslo/messaging/_drivers/pool.py rename to oslo_messaging/_drivers/pool.py diff --git a/oslo/messaging/_drivers/protocols/__init__.py b/oslo_messaging/_drivers/protocols/__init__.py similarity index 100% rename from oslo/messaging/_drivers/protocols/__init__.py rename to oslo_messaging/_drivers/protocols/__init__.py diff --git a/oslo/messaging/_drivers/protocols/amqp/__init__.py b/oslo_messaging/_drivers/protocols/amqp/__init__.py similarity index 100% rename from oslo/messaging/_drivers/protocols/amqp/__init__.py rename to oslo_messaging/_drivers/protocols/amqp/__init__.py diff --git a/oslo/messaging/_drivers/protocols/amqp/controller.py b/oslo_messaging/_drivers/protocols/amqp/controller.py similarity index 99% rename from oslo/messaging/_drivers/protocols/amqp/controller.py rename to oslo_messaging/_drivers/protocols/amqp/controller.py index 7f013533e..ea0950e6d 100644 --- a/oslo/messaging/_drivers/protocols/amqp/controller.py +++ b/oslo_messaging/_drivers/protocols/amqp/controller.py @@ -34,9 +34,9 @@ import pyngus from six import moves from oslo.config import cfg -from oslo.messaging._drivers.protocols.amqp import eventloop -from oslo.messaging._drivers.protocols.amqp import opts -from oslo.messaging import transport +from oslo_messaging._drivers.protocols.amqp import eventloop +from oslo_messaging._drivers.protocols.amqp import opts +from oslo_messaging import transport LOG = logging.getLogger(__name__) diff --git a/oslo/messaging/_drivers/protocols/amqp/driver.py b/oslo_messaging/_drivers/protocols/amqp/driver.py similarity index 97% rename from oslo/messaging/_drivers/protocols/amqp/driver.py rename to oslo_messaging/_drivers/protocols/amqp/driver.py index e60be6593..8ec8b9e44 100644 --- a/oslo/messaging/_drivers/protocols/amqp/driver.py +++ b/oslo_messaging/_drivers/protocols/amqp/driver.py @@ -28,12 +28,12 @@ import time import proton from six import moves -from oslo import messaging -from oslo.messaging._drivers import base -from oslo.messaging._drivers import common -from oslo.messaging._drivers.protocols.amqp import controller -from oslo.messaging import target as messaging_target from oslo.serialization import jsonutils +import oslo_messaging +from oslo_messaging._drivers import base +from oslo_messaging._drivers import common +from oslo_messaging._drivers.protocols.amqp import controller +from oslo_messaging import target as messaging_target LOG = logging.getLogger(__name__) @@ -67,7 +67,8 @@ class SendTask(controller.Task): try: return self._reply_queue.get(timeout=timeout) except moves.queue.Empty: - raise messaging.MessagingTimeout('Timed out waiting for a reply') + raise oslo_messaging.MessagingTimeout( + 'Timed out waiting for a reply') class ListenTask(controller.Task): diff --git a/oslo/messaging/_drivers/protocols/amqp/eventloop.py b/oslo_messaging/_drivers/protocols/amqp/eventloop.py similarity index 100% rename from oslo/messaging/_drivers/protocols/amqp/eventloop.py rename to oslo_messaging/_drivers/protocols/amqp/eventloop.py diff --git a/oslo/messaging/_drivers/protocols/amqp/opts.py b/oslo_messaging/_drivers/protocols/amqp/opts.py similarity index 100% rename from oslo/messaging/_drivers/protocols/amqp/opts.py rename to oslo_messaging/_drivers/protocols/amqp/opts.py diff --git a/oslo/messaging/_executors/__init__.py b/oslo_messaging/_executors/__init__.py similarity index 100% rename from oslo/messaging/_executors/__init__.py rename to oslo_messaging/_executors/__init__.py diff --git a/oslo/messaging/_executors/base.py b/oslo_messaging/_executors/base.py similarity index 100% rename from oslo/messaging/_executors/base.py rename to oslo_messaging/_executors/base.py diff --git a/oslo/messaging/_executors/impl_blocking.py b/oslo_messaging/_executors/impl_blocking.py similarity index 95% rename from oslo/messaging/_executors/impl_blocking.py rename to oslo_messaging/_executors/impl_blocking.py index 1b039b991..733403601 100644 --- a/oslo/messaging/_executors/impl_blocking.py +++ b/oslo_messaging/_executors/impl_blocking.py @@ -15,8 +15,8 @@ import logging -from oslo.messaging._executors import base -from oslo.messaging._i18n import _ +from oslo_messaging._executors import base +from oslo_messaging._i18n import _ LOG = logging.getLogger(__name__) diff --git a/oslo/messaging/_executors/impl_eventlet.py b/oslo_messaging/_executors/impl_eventlet.py similarity index 97% rename from oslo/messaging/_executors/impl_eventlet.py rename to oslo_messaging/_executors/impl_eventlet.py index 3d11457e0..95d0a2f7d 100644 --- a/oslo/messaging/_executors/impl_eventlet.py +++ b/oslo_messaging/_executors/impl_eventlet.py @@ -21,9 +21,9 @@ from eventlet.green import threading as greenthreading from eventlet import greenpool import greenlet -from oslo.messaging._executors import base -from oslo.messaging import localcontext from oslo.utils import excutils +from oslo_messaging._executors import base +from oslo_messaging import localcontext LOG = logging.getLogger(__name__) diff --git a/oslo/messaging/_executors/impl_thread.py b/oslo_messaging/_executors/impl_thread.py similarity index 99% rename from oslo/messaging/_executors/impl_thread.py rename to oslo_messaging/_executors/impl_thread.py index 57c03bbe0..2ed4560ba 100644 --- a/oslo/messaging/_executors/impl_thread.py +++ b/oslo_messaging/_executors/impl_thread.py @@ -22,8 +22,8 @@ import threading from concurrent import futures import six -from oslo.messaging._executors import base from oslo.utils import excutils +from oslo_messaging._executors import base class ThreadExecutor(base.PooledExecutorBase): diff --git a/oslo/messaging/_i18n.py b/oslo_messaging/_i18n.py similarity index 100% rename from oslo/messaging/_i18n.py rename to oslo_messaging/_i18n.py diff --git a/oslo/messaging/_utils.py b/oslo_messaging/_utils.py similarity index 100% rename from oslo/messaging/_utils.py rename to oslo_messaging/_utils.py diff --git a/oslo_messaging/conffixture.py b/oslo_messaging/conffixture.py new file mode 100644 index 000000000..c41da3866 --- /dev/null +++ b/oslo_messaging/conffixture.py @@ -0,0 +1,78 @@ + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__all__ = ['ConfFixture'] + +import sys + +import fixtures + + +def _import_opts(conf, module, opts): + __import__(module) + conf.register_opts(getattr(sys.modules[module], opts)) + + +class ConfFixture(fixtures.Fixture): + + """Tweak configuration options for unit testing. + + oslo.messaging registers a number of configuration options, but rather than + directly referencing those options, users of the API should use this + interface for querying and overriding certain configuration options. + + An example usage:: + + self.messaging_conf = self.useFixture(messaging.ConfFixture(cfg.CONF)) + self.messaging_conf.transport_driver = 'fake' + + :param conf: a ConfigOpts instance + :type conf: oslo.config.cfg.ConfigOpts + """ + + def __init__(self, conf): + self.conf = conf + _import_opts(self.conf, + 'oslo_messaging._drivers.impl_rabbit', 'rabbit_opts') + _import_opts(self.conf, + 'oslo_messaging._drivers.impl_qpid', 'qpid_opts') + _import_opts(self.conf, + 'oslo_messaging._drivers.amqp', 'amqp_opts') + _import_opts(self.conf, 'oslo_messaging.rpc.client', '_client_opts') + _import_opts(self.conf, 'oslo_messaging.transport', '_transport_opts') + _import_opts(self.conf, + 'oslo_messaging.notify.notifier', '_notifier_opts') + + def setUp(self): + super(ConfFixture, self).setUp() + self.addCleanup(self.conf.reset) + + @property + def transport_driver(self): + """The transport driver - for example 'rabbit', 'qpid' or 'fake'.""" + return self.conf.rpc_backend + + @transport_driver.setter + def transport_driver(self, value): + self.conf.set_override('rpc_backend', value) + + @property + def response_timeout(self): + """Default number of seconds to wait for a response from a call.""" + return self.conf.rpc_response_timeout + + @response_timeout.setter + def response_timeout(self, value): + self.conf.set_override('rpc_response_timeout', value) diff --git a/oslo_messaging/exceptions.py b/oslo_messaging/exceptions.py new file mode 100644 index 000000000..93f525abe --- /dev/null +++ b/oslo_messaging/exceptions.py @@ -0,0 +1,40 @@ + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__all__ = ['MessagingException', 'MessagingTimeout', 'MessageDeliveryFailure', + 'InvalidTarget'] + +import six + + +class MessagingException(Exception): + """Base class for exceptions.""" + + +class MessagingTimeout(MessagingException): + """Raised if message sending times out.""" + + +class MessageDeliveryFailure(MessagingException): + """Raised if message sending failed after the asked retry.""" + + +class InvalidTarget(MessagingException, ValueError): + """Raised if a target does not meet certain pre-conditions.""" + + def __init__(self, msg, target): + msg = msg + ":" + six.text_type(target) + super(InvalidTarget, self).__init__(msg) + self.target = target diff --git a/oslo_messaging/localcontext.py b/oslo_messaging/localcontext.py new file mode 100644 index 000000000..8331152d5 --- /dev/null +++ b/oslo_messaging/localcontext.py @@ -0,0 +1,55 @@ + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__all__ = [ + 'get_local_context', + 'set_local_context', + 'clear_local_context', +] + +import threading +import uuid + +_KEY = '_%s_%s' % (__name__.replace('.', '_'), uuid.uuid4().hex) +_STORE = threading.local() + + +def get_local_context(ctxt): + """Retrieve the RPC endpoint request context for the current thread. + + This method allows any code running in the context of a dispatched RPC + endpoint method to retrieve the context for this request. + + This is commonly used for logging so that, for example, you can include the + request ID, user and tenant in every message logged from a RPC endpoint + method. + + :returns: the context for the request dispatched in the current thread + """ + return getattr(_STORE, _KEY, None) + + +def set_local_context(ctxt): + """Set the request context for the current thread. + + :param ctxt: a deserialized request context + :type ctxt: dict + """ + setattr(_STORE, _KEY, ctxt) + + +def clear_local_context(): + """Clear the request context for the current thread.""" + delattr(_STORE, _KEY) diff --git a/oslo_messaging/notify/__init__.py b/oslo_messaging/notify/__init__.py new file mode 100644 index 000000000..c5032db83 --- /dev/null +++ b/oslo_messaging/notify/__init__.py @@ -0,0 +1,27 @@ + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__all__ = ['Notifier', + 'LoggingNotificationHandler', + 'get_notification_listener', + 'NotificationResult', + 'PublishErrorsHandler', + 'LoggingErrorNotificationHandler'] + +from .notifier import * +from .listener import * +from .log_handler import * +from .logger import * +from .dispatcher import NotificationResult diff --git a/oslo/messaging/notify/_impl_log.py b/oslo_messaging/notify/_impl_log.py similarity index 79% rename from oslo/messaging/notify/_impl_log.py rename to oslo_messaging/notify/_impl_log.py index d03643c31..e04aa618d 100644 --- a/oslo/messaging/notify/_impl_log.py +++ b/oslo_messaging/notify/_impl_log.py @@ -17,14 +17,19 @@ import logging -from oslo.messaging.notify import notifier from oslo.serialization import jsonutils +from oslo_messaging.notify import notifier class LogDriver(notifier._Driver): "Publish notifications via Python logging infrastructure." + # NOTE(dhellmann): For backwards-compatibility with configurations + # that may have modified the settings for this logger using a + # configuration file, we keep the name + # 'oslo.messaging.notification' even though the package is now + # 'oslo_messaging'. LOGGER_BASE = 'oslo.messaging.notification' def notify(self, ctxt, message, priority, retry): diff --git a/oslo/messaging/notify/_impl_messaging.py b/oslo_messaging/notify/_impl_messaging.py similarity index 93% rename from oslo/messaging/notify/_impl_messaging.py rename to oslo_messaging/notify/_impl_messaging.py index e562f79ae..f7cef37a1 100644 --- a/oslo/messaging/notify/_impl_messaging.py +++ b/oslo_messaging/notify/_impl_messaging.py @@ -17,8 +17,8 @@ import logging -from oslo import messaging -from oslo.messaging.notify import notifier +import oslo_messaging +from oslo_messaging.notify import notifier LOG = logging.getLogger(__name__) @@ -41,7 +41,7 @@ class MessagingDriver(notifier._Driver): def notify(self, ctxt, message, priority, retry): priority = priority.lower() for topic in self.topics: - target = messaging.Target(topic='%s.%s' % (topic, priority)) + target = oslo_messaging.Target(topic='%s.%s' % (topic, priority)) try: self.transport._send_notification(target, ctxt, message, version=self.version, diff --git a/oslo/messaging/notify/_impl_noop.py b/oslo_messaging/notify/_impl_noop.py similarity index 94% rename from oslo/messaging/notify/_impl_noop.py rename to oslo_messaging/notify/_impl_noop.py index e6acb62d8..232c90406 100644 --- a/oslo/messaging/notify/_impl_noop.py +++ b/oslo_messaging/notify/_impl_noop.py @@ -15,7 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo.messaging.notify import notifier +from oslo_messaging.notify import notifier class NoOpDriver(notifier._Driver): diff --git a/oslo/messaging/notify/_impl_routing.py b/oslo_messaging/notify/_impl_routing.py similarity index 98% rename from oslo/messaging/notify/_impl_routing.py rename to oslo_messaging/notify/_impl_routing.py index 5b879424d..666ad4de4 100644 --- a/oslo/messaging/notify/_impl_routing.py +++ b/oslo_messaging/notify/_impl_routing.py @@ -21,8 +21,8 @@ from stevedore import dispatch import yaml from oslo.config import cfg -from oslo.messaging._i18n import _ -from oslo.messaging.notify import notifier +from oslo_messaging._i18n import _ +from oslo_messaging.notify import notifier LOG = logging.getLogger(__name__) diff --git a/oslo/messaging/notify/_impl_test.py b/oslo_messaging/notify/_impl_test.py similarity index 95% rename from oslo/messaging/notify/_impl_test.py rename to oslo_messaging/notify/_impl_test.py index 0c861d157..89501d611 100644 --- a/oslo/messaging/notify/_impl_test.py +++ b/oslo_messaging/notify/_impl_test.py @@ -15,7 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo.messaging.notify import notifier +from oslo_messaging.notify import notifier NOTIFICATIONS = [] diff --git a/oslo_messaging/notify/dispatcher.py b/oslo_messaging/notify/dispatcher.py new file mode 100644 index 000000000..a9e8cc618 --- /dev/null +++ b/oslo_messaging/notify/dispatcher.py @@ -0,0 +1,135 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# Copyright 2013 eNovance +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import contextlib +import itertools +import logging +import sys + +from oslo_messaging import localcontext +from oslo_messaging import serializer as msg_serializer + + +LOG = logging.getLogger(__name__) + +PRIORITIES = ['audit', 'debug', 'info', 'warn', 'error', 'critical', 'sample'] + + +class NotificationResult(object): + HANDLED = 'handled' + REQUEUE = 'requeue' + + +class NotificationDispatcher(object): + """A message dispatcher which understands Notification messages. + + A MessageHandlingServer is constructed by passing a callable dispatcher + which is invoked with context and message dictionaries each time a message + is received. + + NotifcationDispatcher is one such dispatcher which pass a raw notification + message to the endpoints + """ + + def __init__(self, targets, endpoints, serializer, allow_requeue, + pool=None): + self.targets = targets + self.endpoints = endpoints + self.serializer = serializer or msg_serializer.NoOpSerializer() + self.allow_requeue = allow_requeue + self.pool = pool + + self._callbacks_by_priority = {} + for endpoint, prio in itertools.product(endpoints, PRIORITIES): + if hasattr(endpoint, prio): + method = getattr(endpoint, prio) + self._callbacks_by_priority.setdefault(prio, []).append(method) + + priorities = self._callbacks_by_priority.keys() + self._targets_priorities = set(itertools.product(self.targets, + priorities)) + + def _listen(self, transport): + return transport._listen_for_notifications(self._targets_priorities, + pool=self.pool) + + @contextlib.contextmanager + def __call__(self, incoming, executor_callback=None): + result_wrapper = [] + + yield lambda: result_wrapper.append( + self._dispatch_and_handle_error(incoming, executor_callback)) + + if result_wrapper[0] == NotificationResult.HANDLED: + incoming.acknowledge() + else: + incoming.requeue() + + def _dispatch_and_handle_error(self, incoming, executor_callback): + """Dispatch a notification message to the appropriate endpoint method. + + :param incoming: the incoming notification message + :type ctxt: IncomingMessage + """ + try: + return self._dispatch(incoming.ctxt, incoming.message, + executor_callback) + except Exception: + # sys.exc_info() is deleted by LOG.exception(). + exc_info = sys.exc_info() + LOG.error('Exception during message handling', + exc_info=exc_info) + return NotificationResult.HANDLED + + def _dispatch(self, ctxt, message, executor_callback=None): + """Dispatch an RPC message to the appropriate endpoint method. + + :param ctxt: the request context + :type ctxt: dict + :param message: the message payload + :type message: dict + """ + ctxt = self.serializer.deserialize_context(ctxt) + + publisher_id = message.get('publisher_id') + event_type = message.get('event_type') + metadata = { + 'message_id': message.get('message_id'), + 'timestamp': message.get('timestamp') + } + priority = message.get('priority', '').lower() + if priority not in PRIORITIES: + LOG.warning('Unknown priority "%s"', priority) + return + + payload = self.serializer.deserialize_entity(ctxt, + message.get('payload')) + + for callback in self._callbacks_by_priority.get(priority, []): + localcontext.set_local_context(ctxt) + try: + if executor_callback: + ret = executor_callback(callback, ctxt, publisher_id, + event_type, payload, metadata) + else: + ret = callback(ctxt, publisher_id, event_type, payload, + metadata) + ret = NotificationResult.HANDLED if ret is None else ret + if self.allow_requeue and ret == NotificationResult.REQUEUE: + return ret + finally: + localcontext.clear_local_context() + return NotificationResult.HANDLED diff --git a/oslo_messaging/notify/listener.py b/oslo_messaging/notify/listener.py new file mode 100644 index 000000000..a1586ddfb --- /dev/null +++ b/oslo_messaging/notify/listener.py @@ -0,0 +1,137 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# Copyright 2013 eNovance +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""A notification listener exposes a number of endpoints, each of which +contain a set of methods. Each method corresponds to a notification priority. + +To create a notification listener, you supply a transport, list of targets and +a list of endpoints. + +A transport can be obtained simply by calling the get_transport() method:: + + transport = messaging.get_transport(conf) + +which will load the appropriate transport driver according to the user's +messaging configuration configuration. See get_transport() for more details. + +The target supplied when creating a notification listener expresses the topic +and - optionally - the exchange to listen on. See Target for more details +on these attributes. + +Notification listener have start(), stop() and wait() messages to begin +handling requests, stop handling requests and wait for all in-process +requests to complete. + +Each notification listener is associated with an executor which integrates the +listener with a specific I/O handling framework. Currently, there are blocking +and eventlet executors available. + +A simple example of a notification listener with multiple endpoints might be:: + + from oslo.config import cfg + import oslo_messaging + + class NotificationEndpoint(object): + def warn(self, ctxt, publisher_id, event_type, payload, metadata): + do_something(payload) + + class ErrorEndpoint(object): + def error(self, ctxt, publisher_id, event_type, payload, metadata): + do_something(payload) + + transport = oslo_messaging.get_transport(cfg.CONF) + targets = [ + oslo_messaging.Target(topic='notifications') + oslo_messaging.Target(topic='notifications_bis') + ] + endpoints = [ + NotificationEndpoint(), + ErrorEndpoint(), + ] + pool = "listener-workers" + server = oslo_messaging.get_notification_listener(transport, targets, + endpoints, pool) + server.start() + server.wait() + +A notifier sends a notification on a topic with a priority, the notification +listener will receive this notification if the topic of this one have been set +in one of the targets and if an endpoint implements the method named like the +priority + +Parameters to endpoint methods are the request context supplied by the client, +the publisher_id of the notification message, the event_type, the payload and +metadata. The metadata parameter is a mapping containing a unique message_id +and a timestamp. + +By supplying a serializer object, a listener can deserialize a request context +and arguments from - and serialize return values to - primitive types. + +By supplying a pool name you can create multiple groups of listeners consuming +notifications and that each group only receives one copy of each +notification. + +An endpoint method can explicitly return +oslo_messaging.NotificationResult.HANDLED to acknowledge a message or +oslo_messaging.NotificationResult.REQUEUE to requeue the message. + +The message is acknowledged only if all endpoints either return +oslo_messaging.NotificationResult.HANDLED or None. + +Note that not all transport drivers implement support for requeueing. In order +to use this feature, applications should assert that the feature is available +by passing allow_requeue=True to get_notification_listener(). If the driver +does not support requeueing, it will raise NotImplementedError at this point. + +""" + +from oslo_messaging.notify import dispatcher as notify_dispatcher +from oslo_messaging import server as msg_server + + +def get_notification_listener(transport, targets, endpoints, + executor='blocking', serializer=None, + allow_requeue=False, pool=None): + """Construct a notification listener + + The executor parameter controls how incoming messages will be received and + dispatched. By default, the most simple executor is used - the blocking + executor. + + If the eventlet executor is used, the threading and time library need to be + monkeypatched. + + :param transport: the messaging transport + :type transport: Transport + :param targets: the exchanges and topics to listen on + :type targets: list of Target + :param endpoints: a list of endpoint objects + :type endpoints: list + :param executor: name of a message executor - for example + 'eventlet', 'blocking' + :type executor: str + :param serializer: an optional entity serializer + :type serializer: Serializer + :param allow_requeue: whether NotificationResult.REQUEUE support is needed + :type allow_requeue: bool + :param pool: the pool name + :type pool: str + :raises: NotImplementedError + """ + transport._require_driver_features(requeue=allow_requeue) + dispatcher = notify_dispatcher.NotificationDispatcher(targets, endpoints, + serializer, + allow_requeue, pool) + return msg_server.MessageHandlingServer(transport, dispatcher, executor) diff --git a/oslo_messaging/notify/log_handler.py b/oslo_messaging/notify/log_handler.py new file mode 100644 index 000000000..dbbc67f91 --- /dev/null +++ b/oslo_messaging/notify/log_handler.py @@ -0,0 +1,42 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + +from oslo.config import cfg + + +class LoggingErrorNotificationHandler(logging.Handler): + def __init__(self, *args, **kwargs): + # NOTE(dhellmann): Avoid a cyclical import by doing this one + # at runtime. + import oslo_messaging + logging.Handler.__init__(self, *args, **kwargs) + self._transport = oslo_messaging.get_transport(cfg.CONF) + self._notifier = oslo_messaging.Notifier( + self._transport, + publisher_id='error.publisher') + + def emit(self, record): + # NOTE(bnemec): Notifier registers this opt with the transport. + if ('log' in self._transport.conf.notification_driver): + # NOTE(lbragstad): If we detect that log is one of the + # notification drivers, then return. This protects from infinite + # recursion where something bad happens, it gets logged, the log + # handler sends a notification, and the log_notifier sees the + # notification and logs it. + return + self._notifier.error(None, 'error_notification', + dict(error=record.msg)) + + +PublishErrorsHandler = LoggingErrorNotificationHandler diff --git a/oslo_messaging/notify/logger.py b/oslo_messaging/notify/logger.py new file mode 100644 index 000000000..eb8e4458b --- /dev/null +++ b/oslo_messaging/notify/logger.py @@ -0,0 +1,81 @@ +# Copyright 2013 eNovance +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Driver for the Python logging package that sends log records as a notification. +""" +import logging + +from oslo.config import cfg +from oslo_messaging.notify import notifier +from oslo_messaging import transport + + +class LoggingNotificationHandler(logging.Handler): + """Handler for logging to the messaging notification system. + + Each time the application logs a message using the :py:mod:`logging` + module, it will be sent as a notification. The severity used for the + notification will be the same as the one used for the log record. + + This can be used into a Python logging configuration this way:: + + [handler_notifier] + class=oslo_messaging.LoggingNotificationHandler + level=ERROR + args=('qpid:///') + + """ + + CONF = cfg.CONF + """Default configuration object used, subclass this class if you want to + use another one. + + """ + + def __init__(self, url, publisher_id=None, driver=None, + topic=None, serializer=None): + self.notifier = notifier.Notifier( + transport.get_transport(self.CONF, url), + publisher_id, driver, + topic, + serializer() if serializer else None) + logging.Handler.__init__(self) + + def emit(self, record): + """Emit the log record to the messaging notification system. + + :param record: A log record to emit. + + """ + method = getattr(self.notifier, record.levelname.lower(), None) + + if not method: + return + + method(None, + 'logrecord', + { + 'name': record.name, + 'levelno': record.levelno, + 'levelname': record.levelname, + 'exc_info': record.exc_info, + 'pathname': record.pathname, + 'lineno': record.lineno, + 'msg': record.getMessage(), + 'funcName': record.funcName, + 'thread': record.thread, + 'processName': record.processName, + 'process': record.process, + 'extra': getattr(record, 'extra', None), + }) diff --git a/oslo_messaging/notify/middleware.py b/oslo_messaging/notify/middleware.py new file mode 100644 index 000000000..5529713c0 --- /dev/null +++ b/oslo_messaging/notify/middleware.py @@ -0,0 +1,128 @@ +# Copyright (c) 2013-2014 eNovance +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Send notifications on request + +""" +import logging +import os.path +import sys +import traceback as tb + +import six +import webob.dec + +from oslo.config import cfg +from oslo.middleware import base +import oslo_messaging +from oslo_messaging._i18n import _LE +from oslo_messaging import notify +from oslo_messaging.openstack.common import context + +LOG = logging.getLogger(__name__) + + +def log_and_ignore_error(fn): + def wrapped(*args, **kwargs): + try: + return fn(*args, **kwargs) + except Exception as e: + LOG.exception(_LE('An exception occurred processing ' + 'the API call: %s ') % e) + return wrapped + + +class RequestNotifier(base.Middleware): + """Send notification on request.""" + + @classmethod + def factory(cls, global_conf, **local_conf): + """Factory method for paste.deploy.""" + conf = global_conf.copy() + conf.update(local_conf) + + def _factory(app): + return cls(app, **conf) + return _factory + + def __init__(self, app, **conf): + self.notifier = notify.Notifier( + oslo_messaging.get_transport(cfg.CONF, conf.get('url')), + publisher_id=conf.get('publisher_id', + os.path.basename(sys.argv[0]))) + self.service_name = conf.get('service_name') + self.ignore_req_list = [x.upper().strip() for x in + conf.get('ignore_req_list', '').split(',')] + super(RequestNotifier, self).__init__(app) + + @staticmethod + def environ_to_dict(environ): + """Following PEP 333, server variables are lower case, so don't + include them. + + """ + return dict((k, v) for k, v in six.iteritems(environ) + if k.isupper() and k != 'HTTP_X_AUTH_TOKEN') + + @log_and_ignore_error + def process_request(self, request): + request.environ['HTTP_X_SERVICE_NAME'] = \ + self.service_name or request.host + payload = { + 'request': self.environ_to_dict(request.environ), + } + + self.notifier.info(context.get_admin_context(), + 'http.request', + payload) + + @log_and_ignore_error + def process_response(self, request, response, + exception=None, traceback=None): + payload = { + 'request': self.environ_to_dict(request.environ), + } + + if response: + payload['response'] = { + 'status': response.status, + 'headers': response.headers, + } + + if exception: + payload['exception'] = { + 'value': repr(exception), + 'traceback': tb.format_tb(traceback) + } + + self.notifier.info(context.get_admin_context(), + 'http.response', + payload) + + @webob.dec.wsgify + def __call__(self, req): + if req.method in self.ignore_req_list: + return req.get_response(self.application) + else: + self.process_request(req) + try: + response = req.get_response(self.application) + except Exception: + exc_type, value, traceback = sys.exc_info() + self.process_response(req, None, value, traceback) + raise + else: + self.process_response(req, response) + return response diff --git a/oslo_messaging/notify/notifier.py b/oslo_messaging/notify/notifier.py new file mode 100644 index 000000000..7cc3d9c21 --- /dev/null +++ b/oslo_messaging/notify/notifier.py @@ -0,0 +1,315 @@ + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import logging +import uuid + +import six +from stevedore import named + +from oslo.config import cfg +from oslo.utils import timeutils +from oslo_messaging import serializer as msg_serializer + +_notifier_opts = [ + cfg.MultiStrOpt('notification_driver', + default=[], + help='Driver or drivers to handle sending notifications.'), + cfg.ListOpt('notification_topics', + default=['notifications', ], + deprecated_name='topics', + deprecated_group='rpc_notifier2', + help='AMQP topic used for OpenStack notifications.'), +] + +_LOG = logging.getLogger(__name__) + + +@six.add_metaclass(abc.ABCMeta) +class _Driver(object): + + def __init__(self, conf, topics, transport): + self.conf = conf + self.topics = topics + self.transport = transport + + @abc.abstractmethod + def notify(self, ctxt, msg, priority, retry): + pass + + +class Notifier(object): + + """Send notification messages. + + The Notifier class is used for sending notification messages over a + messaging transport or other means. + + Notification messages follow the following format:: + + {'message_id': six.text_type(uuid.uuid4()), + 'publisher_id': 'compute.host1', + 'timestamp': timeutils.utcnow(), + 'priority': 'WARN', + 'event_type': 'compute.create_instance', + 'payload': {'instance_id': 12, ... }} + + A Notifier object can be instantiated with a transport object and a + publisher ID: + + notifier = messaging.Notifier(get_transport(CONF), 'compute') + + and notifications are sent via drivers chosen with the notification_driver + config option and on the topics chosen with the notification_topics config + option. + + Alternatively, a Notifier object can be instantiated with a specific + driver or topic:: + + notifier = notifier.Notifier(RPC_TRANSPORT, + 'compute.host', + driver='messaging', + topic='notifications') + + Notifier objects are relatively expensive to instantiate (mostly the cost + of loading notification drivers), so it is possible to specialize a given + Notifier object with a different publisher id using the prepare() method:: + + notifier = notifier.prepare(publisher_id='compute') + notifier.info(ctxt, event_type, payload) + """ + + def __init__(self, transport, publisher_id=None, + driver=None, topic=None, + serializer=None, retry=None): + """Construct a Notifier object. + + :param transport: the transport to use for sending messages + :type transport: oslo_messaging.Transport + :param publisher_id: field in notifications sent, for example + 'compute.host1' + :type publisher_id: str + :param driver: a driver to lookup from oslo_messaging.notify.drivers + :type driver: str + :param topic: the topic which to send messages on + :type topic: str + :param serializer: an optional entity serializer + :type serializer: Serializer + :param retry: an connection retries configuration + None or -1 means to retry forever + 0 means no retry + N means N retries + :type retry: int + """ + transport.conf.register_opts(_notifier_opts) + + self.transport = transport + self.publisher_id = publisher_id + self.retry = retry + + self._driver_names = ([driver] if driver is not None + else transport.conf.notification_driver) + + self._topics = ([topic] if topic is not None + else transport.conf.notification_topics) + self._serializer = serializer or msg_serializer.NoOpSerializer() + + self._driver_mgr = named.NamedExtensionManager( + 'oslo.messaging.notify.drivers', + names=self._driver_names, + invoke_on_load=True, + invoke_args=[transport.conf], + invoke_kwds={ + 'topics': self._topics, + 'transport': self.transport, + } + ) + + _marker = object() + + def prepare(self, publisher_id=_marker, retry=_marker): + """Return a specialized Notifier instance. + + Returns a new Notifier instance with the supplied publisher_id. Allows + sending notifications from multiple publisher_ids without the overhead + of notification driver loading. + + :param publisher_id: field in notifications sent, for example + 'compute.host1' + :type publisher_id: str + :param retry: an connection retries configuration + None or -1 means to retry forever + 0 means no retry + N means N retries + :type retry: int + """ + return _SubNotifier._prepare(self, publisher_id, retry=retry) + + def _notify(self, ctxt, event_type, payload, priority, publisher_id=None, + retry=None): + payload = self._serializer.serialize_entity(ctxt, payload) + ctxt = self._serializer.serialize_context(ctxt) + + msg = dict(message_id=six.text_type(uuid.uuid4()), + publisher_id=publisher_id or self.publisher_id, + event_type=event_type, + priority=priority, + payload=payload, + timestamp=six.text_type(timeutils.utcnow())) + + def do_notify(ext): + try: + ext.obj.notify(ctxt, msg, priority, retry or self.retry) + except Exception as e: + _LOG.exception("Problem '%(e)s' attempting to send to " + "notification system. Payload=%(payload)s", + dict(e=e, payload=payload)) + + if self._driver_mgr.extensions: + self._driver_mgr.map(do_notify) + + def audit(self, ctxt, event_type, payload): + """Send a notification at audit level. + + :param ctxt: a request context dict + :type ctxt: dict + :param event_type: describes the event, for example + 'compute.create_instance' + :type event_type: str + :param payload: the notification payload + :type payload: dict + :raises: MessageDeliveryFailure + """ + self._notify(ctxt, event_type, payload, 'AUDIT') + + def debug(self, ctxt, event_type, payload): + """Send a notification at debug level. + + :param ctxt: a request context dict + :type ctxt: dict + :param event_type: describes the event, for example + 'compute.create_instance' + :type event_type: str + :param payload: the notification payload + :type payload: dict + :raises: MessageDeliveryFailure + """ + self._notify(ctxt, event_type, payload, 'DEBUG') + + def info(self, ctxt, event_type, payload): + """Send a notification at info level. + + :param ctxt: a request context dict + :type ctxt: dict + :param event_type: describes the event, for example + 'compute.create_instance' + :type event_type: str + :param payload: the notification payload + :type payload: dict + :raises: MessageDeliveryFailure + """ + self._notify(ctxt, event_type, payload, 'INFO') + + def warn(self, ctxt, event_type, payload): + """Send a notification at warning level. + + :param ctxt: a request context dict + :type ctxt: dict + :param event_type: describes the event, for example + 'compute.create_instance' + :type event_type: str + :param payload: the notification payload + :type payload: dict + :raises: MessageDeliveryFailure + """ + self._notify(ctxt, event_type, payload, 'WARN') + + warning = warn + + def error(self, ctxt, event_type, payload): + """Send a notification at error level. + + :param ctxt: a request context dict + :type ctxt: dict + :param event_type: describes the event, for example + 'compute.create_instance' + :type event_type: str + :param payload: the notification payload + :type payload: dict + :raises: MessageDeliveryFailure + """ + self._notify(ctxt, event_type, payload, 'ERROR') + + def critical(self, ctxt, event_type, payload): + """Send a notification at critical level. + + :param ctxt: a request context dict + :type ctxt: dict + :param event_type: describes the event, for example + 'compute.create_instance' + :type event_type: str + :param payload: the notification payload + :type payload: dict + :raises: MessageDeliveryFailure + """ + self._notify(ctxt, event_type, payload, 'CRITICAL') + + def sample(self, ctxt, event_type, payload): + """Send a notification at sample level. + + Sample notifications are for high-frequency events + that typically contain small payloads. eg: "CPU = 70%" + + Not all drivers support the sample level + (log, for example) so these could be dropped. + + :param ctxt: a request context dict + :type ctxt: dict + :param event_type: describes the event, for example + 'compute.create_instance' + :type event_type: str + :param payload: the notification payload + :type payload: dict + :raises: MessageDeliveryFailure + """ + self._notify(ctxt, event_type, payload, 'SAMPLE') + + +class _SubNotifier(Notifier): + + _marker = Notifier._marker + + def __init__(self, base, publisher_id, retry): + self._base = base + self.transport = base.transport + self.publisher_id = publisher_id + self.retry = retry + + self._serializer = self._base._serializer + self._driver_mgr = self._base._driver_mgr + + def _notify(self, ctxt, event_type, payload, priority): + super(_SubNotifier, self)._notify(ctxt, event_type, payload, priority) + + @classmethod + def _prepare(cls, base, publisher_id=_marker, retry=_marker): + if publisher_id is cls._marker: + publisher_id = base.publisher_id + if retry is cls._marker: + retry = base.retry + return cls(base, publisher_id, retry=retry) diff --git a/oslo/messaging/openstack/__init__.py b/oslo_messaging/openstack/__init__.py similarity index 100% rename from oslo/messaging/openstack/__init__.py rename to oslo_messaging/openstack/__init__.py diff --git a/oslo/messaging/openstack/common/__init__.py b/oslo_messaging/openstack/common/__init__.py similarity index 100% rename from oslo/messaging/openstack/common/__init__.py rename to oslo_messaging/openstack/common/__init__.py diff --git a/oslo/messaging/openstack/common/context.py b/oslo_messaging/openstack/common/context.py similarity index 100% rename from oslo/messaging/openstack/common/context.py rename to oslo_messaging/openstack/common/context.py diff --git a/oslo/messaging/opts.py b/oslo_messaging/opts.py similarity index 75% rename from oslo/messaging/opts.py rename to oslo_messaging/opts.py index 41264c239..400846529 100644 --- a/oslo/messaging/opts.py +++ b/oslo_messaging/opts.py @@ -20,18 +20,18 @@ __all__ = [ import copy import itertools -from oslo.messaging._drivers import amqp -from oslo.messaging._drivers import impl_qpid -from oslo.messaging._drivers import impl_rabbit -from oslo.messaging._drivers import impl_zmq -from oslo.messaging._drivers import matchmaker -from oslo.messaging._drivers import matchmaker_redis -from oslo.messaging._drivers import matchmaker_ring -from oslo.messaging._drivers.protocols.amqp import opts as amqp_opts -from oslo.messaging._executors import base -from oslo.messaging.notify import notifier -from oslo.messaging.rpc import client -from oslo.messaging import transport +from oslo_messaging._drivers import amqp +from oslo_messaging._drivers import impl_qpid +from oslo_messaging._drivers import impl_rabbit +from oslo_messaging._drivers import impl_zmq +from oslo_messaging._drivers import matchmaker +from oslo_messaging._drivers import matchmaker_redis +from oslo_messaging._drivers import matchmaker_ring +from oslo_messaging._drivers.protocols.amqp import opts as amqp_opts +from oslo_messaging._executors import base +from oslo_messaging.notify import notifier +from oslo_messaging.rpc import client +from oslo_messaging import transport _global_opt_lists = [ amqp.amqp_opts, @@ -64,7 +64,7 @@ def list_opts(): registered. A group name of None corresponds to the [DEFAULT] group in config files. - This function is also discoverable via the 'oslo.messaging' entry point + This function is also discoverable via the 'oslo_messaging' entry point under the 'oslo.config.opts' namespace. The purpose of this is to allow tools like the Oslo sample config file diff --git a/oslo_messaging/rpc/__init__.py b/oslo_messaging/rpc/__init__.py new file mode 100644 index 000000000..f9cc88194 --- /dev/null +++ b/oslo_messaging/rpc/__init__.py @@ -0,0 +1,32 @@ + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__all__ = [ + 'ClientSendError', + 'ExpectedException', + 'NoSuchMethod', + 'RPCClient', + 'RPCDispatcher', + 'RPCDispatcherError', + 'RPCVersionCapError', + 'RemoteError', + 'UnsupportedVersion', + 'expected_exceptions', + 'get_rpc_server', +] + +from .client import * +from .dispatcher import * +from .server import * diff --git a/oslo_messaging/rpc/client.py b/oslo_messaging/rpc/client.py new file mode 100644 index 000000000..8ac73a15e --- /dev/null +++ b/oslo_messaging/rpc/client.py @@ -0,0 +1,397 @@ + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__all__ = [ + 'ClientSendError', + 'RPCClient', + 'RPCVersionCapError', + 'RemoteError', +] + +import six + +from oslo.config import cfg +from oslo_messaging._drivers import base as driver_base +from oslo_messaging import _utils as utils +from oslo_messaging import exceptions +from oslo_messaging import serializer as msg_serializer + +_client_opts = [ + cfg.IntOpt('rpc_response_timeout', + default=60, + help='Seconds to wait for a response from a call.'), +] + + +class RemoteError(exceptions.MessagingException): + + """Signifies that a remote endpoint method has raised an exception. + + Contains a string representation of the type of the original exception, + the value of the original exception, and the traceback. These are + sent to the parent as a joined string so printing the exception + contains all of the relevant info. + """ + + def __init__(self, exc_type=None, value=None, traceback=None): + self.exc_type = exc_type + self.value = value + self.traceback = traceback + msg = ("Remote error: %(exc_type)s %(value)s\n%(traceback)s." % + dict(exc_type=self.exc_type, value=self.value, + traceback=self.traceback)) + super(RemoteError, self).__init__(msg) + + +class RPCVersionCapError(exceptions.MessagingException): + + def __init__(self, version, version_cap): + self.version = version + self.version_cap = version_cap + msg = ("Requested message version, %(version)s is too high. It needs " + "to be lower than the specified version cap %(version_cap)s." % + dict(version=self.version, version_cap=self.version_cap)) + super(RPCVersionCapError, self).__init__(msg) + + +class ClientSendError(exceptions.MessagingException): + """Raised if we failed to send a message to a target.""" + + def __init__(self, target, ex): + msg = 'Failed to send to target "%s": %s' % (target, ex) + super(ClientSendError, self).__init__(msg) + self.target = target + self.ex = ex + + +class _CallContext(object): + + _marker = object() + + def __init__(self, transport, target, serializer, + timeout=None, version_cap=None, retry=None): + self.conf = transport.conf + + self.transport = transport + self.target = target + self.serializer = serializer + self.timeout = timeout + self.retry = retry + self.version_cap = version_cap + + super(_CallContext, self).__init__() + + def _make_message(self, ctxt, method, args): + msg = dict(method=method) + + msg['args'] = dict() + for argname, arg in six.iteritems(args): + msg['args'][argname] = self.serializer.serialize_entity(ctxt, arg) + + if self.target.namespace is not None: + msg['namespace'] = self.target.namespace + if self.target.version is not None: + msg['version'] = self.target.version + + return msg + + def _check_version_cap(self, version): + if not utils.version_is_compatible(self.version_cap, version): + raise RPCVersionCapError(version=version, + version_cap=self.version_cap) + + def can_send_version(self, version=_marker): + """Check to see if a version is compatible with the version cap.""" + version = self.target.version if version is self._marker else version + return (not self.version_cap or + utils.version_is_compatible(self.version_cap, + self.target.version)) + + def cast(self, ctxt, method, **kwargs): + """Invoke a method and return immediately. See RPCClient.cast().""" + msg = self._make_message(ctxt, method, kwargs) + ctxt = self.serializer.serialize_context(ctxt) + + if self.version_cap: + self._check_version_cap(msg.get('version')) + try: + self.transport._send(self.target, ctxt, msg, retry=self.retry) + except driver_base.TransportDriverError as ex: + raise ClientSendError(self.target, ex) + + def call(self, ctxt, method, **kwargs): + """Invoke a method and wait for a reply. See RPCClient.call().""" + if self.target.fanout: + raise exceptions.InvalidTarget('A call cannot be used with fanout', + self.target) + + msg = self._make_message(ctxt, method, kwargs) + msg_ctxt = self.serializer.serialize_context(ctxt) + + timeout = self.timeout + if self.timeout is None: + timeout = self.conf.rpc_response_timeout + + if self.version_cap: + self._check_version_cap(msg.get('version')) + + try: + result = self.transport._send(self.target, msg_ctxt, msg, + wait_for_reply=True, timeout=timeout, + retry=self.retry) + except driver_base.TransportDriverError as ex: + raise ClientSendError(self.target, ex) + return self.serializer.deserialize_entity(ctxt, result) + + @classmethod + def _prepare(cls, base, + exchange=_marker, topic=_marker, namespace=_marker, + version=_marker, server=_marker, fanout=_marker, + timeout=_marker, version_cap=_marker, retry=_marker): + """Prepare a method invocation context. See RPCClient.prepare().""" + kwargs = dict( + exchange=exchange, + topic=topic, + namespace=namespace, + version=version, + server=server, + fanout=fanout) + kwargs = dict([(k, v) for k, v in kwargs.items() + if v is not cls._marker]) + target = base.target(**kwargs) + + if timeout is cls._marker: + timeout = base.timeout + if retry is cls._marker: + retry = base.retry + if version_cap is cls._marker: + version_cap = base.version_cap + + return _CallContext(base.transport, target, + base.serializer, + timeout, version_cap, retry) + + def prepare(self, exchange=_marker, topic=_marker, namespace=_marker, + version=_marker, server=_marker, fanout=_marker, + timeout=_marker, version_cap=_marker, retry=_marker): + """Prepare a method invocation context. See RPCClient.prepare().""" + return self._prepare(self, + exchange, topic, namespace, + version, server, fanout, + timeout, version_cap, retry) + + +class RPCClient(object): + + """A class for invoking methods on remote servers. + + The RPCClient class is responsible for sending method invocations to remote + servers via a messaging transport. + + A default target is supplied to the RPCClient constructor, but target + attributes can be overridden for individual method invocations using the + prepare() method. + + A method invocation consists of a request context dictionary, a method name + and a dictionary of arguments. A cast() invocation just sends the request + and returns immediately. A call() invocation waits for the server to send + a return value. + + This class is intended to be used by wrapping it in another class which + provides methods on the subclass to perform the remote invocation using + call() or cast():: + + class TestClient(object): + + def __init__(self, transport): + target = messaging.Target(topic='testtopic', version='2.0') + self._client = messaging.RPCClient(transport, target) + + def test(self, ctxt, arg): + return self._client.call(ctxt, 'test', arg=arg) + + An example of using the prepare() method to override some attributes of the + default target:: + + def test(self, ctxt, arg): + cctxt = self._client.prepare(version='2.5') + return cctxt.call(ctxt, 'test', arg=arg) + + RPCClient have a number of other properties - for example, timeout and + version_cap - which may make sense to override for some method invocations, + so they too can be passed to prepare():: + + def test(self, ctxt, arg): + cctxt = self._client.prepare(timeout=10) + return cctxt.call(ctxt, 'test', arg=arg) + + However, this class can be used directly without wrapping it another class. + For example:: + + transport = messaging.get_transport(cfg.CONF) + target = messaging.Target(topic='testtopic', version='2.0') + client = messaging.RPCClient(transport, target) + client.call(ctxt, 'test', arg=arg) + + but this is probably only useful in limited circumstances as a wrapper + class will usually help to make the code much more obvious. + + By default, cast() and call() will block until the message is successfully + sent. However, the retry parameter can be used to have message sending + fail with a MessageDeliveryFailure after the given number of retries. For + example:: + + client = messaging.RPCClient(transport, target, retry=None) + client.call(ctxt, 'sync') + try: + client.prepare(retry=0).cast(ctxt, 'ping') + except messaging.MessageDeliveryFailure: + LOG.error("Failed to send ping message") + """ + + def __init__(self, transport, target, + timeout=None, version_cap=None, serializer=None, retry=None): + """Construct an RPC client. + + :param transport: a messaging transport handle + :type transport: Transport + :param target: the default target for invocations + :type target: Target + :param timeout: an optional default timeout (in seconds) for call()s + :type timeout: int or float + :param version_cap: raise a RPCVersionCapError version exceeds this cap + :type version_cap: str + :param serializer: an optional entity serializer + :type serializer: Serializer + :param retry: an optional default connection retries configuration + None or -1 means to retry forever + 0 means no retry + N means N retries + :type retry: int + """ + self.conf = transport.conf + self.conf.register_opts(_client_opts) + + self.transport = transport + self.target = target + self.timeout = timeout + self.retry = retry + self.version_cap = version_cap + self.serializer = serializer or msg_serializer.NoOpSerializer() + + super(RPCClient, self).__init__() + + _marker = _CallContext._marker + + def prepare(self, exchange=_marker, topic=_marker, namespace=_marker, + version=_marker, server=_marker, fanout=_marker, + timeout=_marker, version_cap=_marker, retry=_marker): + """Prepare a method invocation context. + + Use this method to override client properties for an individual method + invocation. For example:: + + def test(self, ctxt, arg): + cctxt = self.prepare(version='2.5') + return cctxt.call(ctxt, 'test', arg=arg) + + :param exchange: see Target.exchange + :type exchange: str + :param topic: see Target.topic + :type topic: str + :param namespace: see Target.namespace + :type namespace: str + :param version: requirement the server must support, see Target.version + :type version: str + :param server: send to a specific server, see Target.server + :type server: str + :param fanout: send to all servers on topic, see Target.fanout + :type fanout: bool + :param timeout: an optional default timeout (in seconds) for call()s + :type timeout: int or float + :param version_cap: raise a RPCVersionCapError version exceeds this cap + :type version_cap: str + :param retry: an optional connection retries configuration + None or -1 means to retry forever + 0 means no retry + N means N retries + :type retry: int + """ + return _CallContext._prepare(self, + exchange, topic, namespace, + version, server, fanout, + timeout, version_cap, retry) + + def cast(self, ctxt, method, **kwargs): + """Invoke a method and return immediately. + + Method arguments must either be primitive types or types supported by + the client's serializer (if any). + + Similarly, the request context must be a dict unless the client's + serializer supports serializing another type. + + :param ctxt: a request context dict + :type ctxt: dict + :param method: the method name + :type method: str + :param kwargs: a dict of method arguments + :type kwargs: dict + :raises: MessageDeliveryFailure + """ + self.prepare().cast(ctxt, method, **kwargs) + + def call(self, ctxt, method, **kwargs): + """Invoke a method and wait for a reply. + + Method arguments must either be primitive types or types supported by + the client's serializer (if any). Similarly, the request context must + be a dict unless the client's serializer supports serializing another + type. + + The semantics of how any errors raised by the remote RPC endpoint + method are handled are quite subtle. + + Firstly, if the remote exception is contained in one of the modules + listed in the allow_remote_exmods messaging.get_transport() parameter, + then it this exception will be re-raised by call(). However, such + locally re-raised remote exceptions are distinguishable from the same + exception type raised locally because re-raised remote exceptions are + modified such that their class name ends with the '_Remote' suffix so + you may do:: + + if ex.__class__.__name__.endswith('_Remote'): + # Some special case for locally re-raised remote exceptions + + Secondly, if a remote exception is not from a module listed in the + allowed_remote_exmods list, then a messaging.RemoteError exception is + raised with all details of the remote exception. + + :param ctxt: a request context dict + :type ctxt: dict + :param method: the method name + :type method: str + :param kwargs: a dict of method arguments + :type kwargs: dict + :raises: MessagingTimeout, RemoteError, MessageDeliveryFailure + """ + return self.prepare().call(ctxt, method, **kwargs) + + def can_send_version(self, version=_marker): + """Check to see if a version is compatible with the version cap.""" + return self.prepare(version=version).can_send_version() diff --git a/oslo_messaging/rpc/dispatcher.py b/oslo_messaging/rpc/dispatcher.py new file mode 100644 index 000000000..d4a882310 --- /dev/null +++ b/oslo_messaging/rpc/dispatcher.py @@ -0,0 +1,195 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# Copyright 2013 Red Hat, Inc. +# Copyright 2013 New Dream Network, LLC (DreamHost) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__all__ = [ + 'NoSuchMethod', + 'RPCDispatcher', + 'RPCDispatcherError', + 'UnsupportedVersion', + 'ExpectedException', +] + +import contextlib +import logging +import sys + +import six + +from oslo_messaging._i18n import _ +from oslo_messaging import _utils as utils +from oslo_messaging import localcontext +from oslo_messaging import serializer as msg_serializer +from oslo_messaging import server as msg_server +from oslo_messaging import target as msg_target + +LOG = logging.getLogger(__name__) + + +class ExpectedException(Exception): + """Encapsulates an expected exception raised by an RPC endpoint + + Merely instantiating this exception records the current exception + information, which will be passed back to the RPC client without + exceptional logging. + """ + def __init__(self): + self.exc_info = sys.exc_info() + + +class RPCDispatcherError(msg_server.MessagingServerError): + "A base class for all RPC dispatcher exceptions." + + +class NoSuchMethod(RPCDispatcherError, AttributeError): + "Raised if there is no endpoint which exposes the requested method." + + def __init__(self, method): + msg = "Endpoint does not support RPC method %s" % method + super(NoSuchMethod, self).__init__(msg) + self.method = method + + +class UnsupportedVersion(RPCDispatcherError): + "Raised if there is no endpoint which supports the requested version." + + def __init__(self, version, method=None): + msg = "Endpoint does not support RPC version %s" % version + if method: + msg = "%s. Attempted method: %s" % (msg, method) + super(UnsupportedVersion, self).__init__(msg) + self.version = version + self.method = method + + +class RPCDispatcher(object): + """A message dispatcher which understands RPC messages. + + A MessageHandlingServer is constructed by passing a callable dispatcher + which is invoked with context and message dictionaries each time a message + is received. + + RPCDispatcher is one such dispatcher which understands the format of RPC + messages. The dispatcher looks at the namespace, version and method values + in the message and matches those against a list of available endpoints. + + Endpoints may have a target attribute describing the namespace and version + of the methods exposed by that object. All public methods on an endpoint + object are remotely invokable by clients. + + + """ + + def __init__(self, target, endpoints, serializer): + """Construct a rpc server dispatcher. + + :param target: the exchange, topic and server to listen on + :type target: Target + """ + + self.endpoints = endpoints + self.serializer = serializer or msg_serializer.NoOpSerializer() + self._default_target = msg_target.Target() + self._target = target + + def _listen(self, transport): + return transport._listen(self._target) + + @staticmethod + def _is_namespace(target, namespace): + return namespace == target.namespace + + @staticmethod + def _is_compatible(target, version): + endpoint_version = target.version or '1.0' + return utils.version_is_compatible(endpoint_version, version) + + def _do_dispatch(self, endpoint, method, ctxt, args, executor_callback): + ctxt = self.serializer.deserialize_context(ctxt) + new_args = dict() + for argname, arg in six.iteritems(args): + new_args[argname] = self.serializer.deserialize_entity(ctxt, arg) + func = getattr(endpoint, method) + if executor_callback: + result = executor_callback(func, ctxt, **new_args) + else: + result = func(ctxt, **new_args) + return self.serializer.serialize_entity(ctxt, result) + + @contextlib.contextmanager + def __call__(self, incoming, executor_callback=None): + incoming.acknowledge() + yield lambda: self._dispatch_and_reply(incoming, executor_callback) + + def _dispatch_and_reply(self, incoming, executor_callback): + try: + incoming.reply(self._dispatch(incoming.ctxt, + incoming.message, + executor_callback)) + except ExpectedException as e: + LOG.debug(u'Expected exception during message handling (%s)', + e.exc_info[1]) + incoming.reply(failure=e.exc_info, log_failure=False) + except Exception as e: + # sys.exc_info() is deleted by LOG.exception(). + exc_info = sys.exc_info() + LOG.error(_('Exception during message handling: %s'), e, + exc_info=exc_info) + incoming.reply(failure=exc_info) + # NOTE(dhellmann): Remove circular object reference + # between the current stack frame and the traceback in + # exc_info. + del exc_info + + def _dispatch(self, ctxt, message, executor_callback=None): + """Dispatch an RPC message to the appropriate endpoint method. + + :param ctxt: the request context + :type ctxt: dict + :param message: the message payload + :type message: dict + :raises: NoSuchMethod, UnsupportedVersion + """ + method = message.get('method') + args = message.get('args', {}) + namespace = message.get('namespace') + version = message.get('version', '1.0') + + found_compatible = False + for endpoint in self.endpoints: + target = getattr(endpoint, 'target', None) + if not target: + target = self._default_target + + if not (self._is_namespace(target, namespace) and + self._is_compatible(target, version)): + continue + + if hasattr(endpoint, method): + localcontext.set_local_context(ctxt) + try: + return self._do_dispatch(endpoint, method, ctxt, args, + executor_callback) + finally: + localcontext.clear_local_context() + + found_compatible = True + + if found_compatible: + raise NoSuchMethod(method) + else: + raise UnsupportedVersion(version, method=method) diff --git a/oslo_messaging/rpc/server.py b/oslo_messaging/rpc/server.py new file mode 100644 index 000000000..f1cbc1205 --- /dev/null +++ b/oslo_messaging/rpc/server.py @@ -0,0 +1,152 @@ + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +An RPC server exposes a number of endpoints, each of which contain a set of +methods which may be invoked remotely by clients over a given transport. + +To create an RPC server, you supply a transport, target and a list of +endpoints. + +A transport can be obtained simply by calling the get_transport() method:: + + transport = messaging.get_transport(conf) + +which will load the appropriate transport driver according to the user's +messaging configuration configuration. See get_transport() for more details. + +The target supplied when creating an RPC server expresses the topic, server +name and - optionally - the exchange to listen on. See Target for more details +on these attributes. + +Each endpoint object may have a target attribute which may have namespace and +version fields set. By default, we use the 'null namespace' and version 1.0. +Incoming method calls will be dispatched to the first endpoint with the +requested method, a matching namespace and a compatible version number. + +RPC servers have start(), stop() and wait() messages to begin handling +requests, stop handling requests and wait for all in-process requests to +complete. + +A simple example of an RPC server with multiple endpoints might be:: + + from oslo.config import cfg + import oslo_messaging + + class ServerControlEndpoint(object): + + target = oslo_messaging.Target(namespace='control', + version='2.0') + + def __init__(self, server): + self.server = server + + def stop(self, ctx): + if server: + self.server.stop() + + class TestEndpoint(object): + + def test(self, ctx, arg): + return arg + + transport = oslo_messaging.get_transport(cfg.CONF) + target = oslo_messaging.Target(topic='test', server='server1') + endpoints = [ + ServerControlEndpoint(None), + TestEndpoint(), + ] + server = oslo_messaging.get_rpc_server(transport, target, endpoints, + executor='blocking') + server.start() + server.wait() + +Clients can invoke methods on the server by sending the request to a topic and +it gets sent to one of the servers listening on the topic, or by sending the +request to a specific server listening on the topic, or by sending the request +to all servers listening on the topic (known as fanout). These modes are chosen +via the server and fanout attributes on Target but the mode used is transparent +to the server. + +The first parameter to method invocations is always the request context +supplied by the client. + +Parameters to the method invocation are primitive types and so must be the +return values from the methods. By supplying a serializer object, a server can +deserialize a request context and arguments from - and serialize return values +to - primitive types. +""" + +__all__ = [ + 'get_rpc_server', + 'expected_exceptions', +] + +from oslo_messaging.rpc import dispatcher as rpc_dispatcher +from oslo_messaging import server as msg_server + + +def get_rpc_server(transport, target, endpoints, + executor='blocking', serializer=None): + """Construct an RPC server. + + The executor parameter controls how incoming messages will be received and + dispatched. By default, the most simple executor is used - the blocking + executor. + + If the eventlet executor is used, the threading and time library need to be + monkeypatched. + + :param transport: the messaging transport + :type transport: Transport + :param target: the exchange, topic and server to listen on + :type target: Target + :param endpoints: a list of endpoint objects + :type endpoints: list + :param executor: name of a message executor - for example + 'eventlet', 'blocking' + :type executor: str + :param serializer: an optional entity serializer + :type serializer: Serializer + """ + dispatcher = rpc_dispatcher.RPCDispatcher(target, endpoints, serializer) + return msg_server.MessageHandlingServer(transport, dispatcher, executor) + + +def expected_exceptions(*exceptions): + """Decorator for RPC endpoint methods that raise expected exceptions. + + Marking an endpoint method with this decorator allows the declaration + of expected exceptions that the RPC server should not consider fatal, + and not log as if they were generated in a real error scenario. + + Note that this will cause listed exceptions to be wrapped in an + ExpectedException, which is used internally by the RPC sever. The RPC + client will see the original exception type. + """ + def outer(func): + def inner(*args, **kwargs): + try: + return func(*args, **kwargs) + # Take advantage of the fact that we can catch + # multiple exception types using a tuple of + # exception classes, with subclass detection + # for free. Any exception that is not in or + # derived from the args passed to us will be + # ignored and thrown as normal. + except exceptions: + raise rpc_dispatcher.ExpectedException() + return inner + return outer diff --git a/oslo_messaging/serializer.py b/oslo_messaging/serializer.py new file mode 100644 index 000000000..894f0f4a7 --- /dev/null +++ b/oslo_messaging/serializer.py @@ -0,0 +1,76 @@ +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__all__ = ['Serializer', 'NoOpSerializer'] + +"""Provides the definition of a message serialization handler""" + +import abc + +import six + + +@six.add_metaclass(abc.ABCMeta) +class Serializer(object): + """Generic (de-)serialization definition base class.""" + + @abc.abstractmethod + def serialize_entity(self, ctxt, entity): + """Serialize something to primitive form. + + :param ctxt: Request context, in deserialized form + :param entity: Entity to be serialized + :returns: Serialized form of entity + """ + + @abc.abstractmethod + def deserialize_entity(self, ctxt, entity): + """Deserialize something from primitive form. + + :param ctxt: Request context, in deserialized form + :param entity: Primitive to be deserialized + :returns: Deserialized form of entity + """ + + @abc.abstractmethod + def serialize_context(self, ctxt): + """Serialize a request context into a dictionary. + + :param ctxt: Request context + :returns: Serialized form of context + """ + + @abc.abstractmethod + def deserialize_context(self, ctxt): + """Deserialize a dictionary into a request context. + + :param ctxt: Request context dictionary + :returns: Deserialized form of entity + """ + + +class NoOpSerializer(Serializer): + """A serializer that does nothing.""" + + def serialize_entity(self, ctxt, entity): + return entity + + def deserialize_entity(self, ctxt, entity): + return entity + + def serialize_context(self, ctxt): + return ctxt + + def deserialize_context(self, ctxt): + return ctxt diff --git a/oslo_messaging/server.py b/oslo_messaging/server.py new file mode 100644 index 000000000..5aae1e7dd --- /dev/null +++ b/oslo_messaging/server.py @@ -0,0 +1,150 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# Copyright 2013 Red Hat, Inc. +# Copyright 2013 New Dream Network, LLC (DreamHost) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__all__ = [ + 'ExecutorLoadFailure', + 'MessageHandlingServer', + 'MessagingServerError', + 'ServerListenError', +] + +from stevedore import driver + +from oslo_messaging._drivers import base as driver_base +from oslo_messaging import exceptions + + +class MessagingServerError(exceptions.MessagingException): + """Base class for all MessageHandlingServer exceptions.""" + + +class ExecutorLoadFailure(MessagingServerError): + """Raised if an executor can't be loaded.""" + + def __init__(self, executor, ex): + msg = 'Failed to load executor "%s": %s' % (executor, ex) + super(ExecutorLoadFailure, self).__init__(msg) + self.executor = executor + self.ex = ex + + +class ServerListenError(MessagingServerError): + """Raised if we failed to listen on a target.""" + + def __init__(self, target, ex): + msg = 'Failed to listen on target "%s": %s' % (target, ex) + super(ServerListenError, self).__init__(msg) + self.target = target + self.ex = ex + + +class MessageHandlingServer(object): + """Server for handling messages. + + Connect a transport to a dispatcher that knows how to process the + message using an executor that knows how the app wants to create + new tasks. + """ + + def __init__(self, transport, dispatcher, executor='blocking'): + """Construct a message handling server. + + The dispatcher parameter is a callable which is invoked with context + and message dictionaries each time a message is received. + + The executor parameter controls how incoming messages will be received + and dispatched. By default, the most simple executor is used - the + blocking executor. + + :param transport: the messaging transport + :type transport: Transport + :param dispatcher: a callable which is invoked for each method + :type dispatcher: callable + :param executor: name of message executor - for example + 'eventlet', 'blocking' + :type executor: str + """ + self.conf = transport.conf + + self.transport = transport + self.dispatcher = dispatcher + self.executor = executor + + try: + mgr = driver.DriverManager('oslo.messaging.executors', + self.executor) + except RuntimeError as ex: + raise ExecutorLoadFailure(self.executor, ex) + else: + self._executor_cls = mgr.driver + self._executor = None + + super(MessageHandlingServer, self).__init__() + + def start(self): + """Start handling incoming messages. + + This method causes the server to begin polling the transport for + incoming messages and passing them to the dispatcher. Message + processing will continue until the stop() method is called. + + The executor controls how the server integrates with the applications + I/O handling strategy - it may choose to poll for messages in a new + process, thread or co-operatively scheduled coroutine or simply by + registering a callback with an event loop. Similarly, the executor may + choose to dispatch messages in a new thread, coroutine or simply the + current thread. + """ + if self._executor is not None: + return + try: + listener = self.dispatcher._listen(self.transport) + except driver_base.TransportDriverError as ex: + raise ServerListenError(self.target, ex) + + self._executor = self._executor_cls(self.conf, listener, + self.dispatcher) + self._executor.start() + + def stop(self): + """Stop handling incoming messages. + + Once this method returns, no new incoming messages will be handled by + the server. However, the server may still be in the process of handling + some messages, and underlying driver resources associated to this + server are still in use. See 'wait' for more details. + """ + if self._executor is not None: + self._executor.stop() + + def wait(self): + """Wait for message processing to complete. + + After calling stop(), there may still be some some existing messages + which have not been completely processed. The wait() method blocks + until all message processing has completed. + + Once it's finished, the underlying driver resources associated to this + server are released (like closing useless network connections). + """ + if self._executor is not None: + self._executor.wait() + # Close listener connection after processing all messages + self._executor.listener.cleanup() + + self._executor = None diff --git a/oslo_messaging/target.py b/oslo_messaging/target.py new file mode 100644 index 000000000..f37a2b296 --- /dev/null +++ b/oslo_messaging/target.py @@ -0,0 +1,94 @@ + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class Target(object): + + """Identifies the destination of messages. + + A Target encapsulates all the information to identify where a message + should be sent or what messages a server is listening for. + + Different subsets of the information encapsulated in a Target object is + relevant to various aspects of the API: + + creating a server: + topic and server is required; exchange is optional + an endpoint's target: + namespace and version is optional + client sending a message: + topic is required, all other attributes optional + + Its attributes are: + + :param exchange: A scope for topics. Leave unspecified to default to the + control_exchange configuration option. + :type exchange: str + :param topic: A name which identifies the set of interfaces exposed by a + server. Multiple servers may listen on a topic and messages will be + dispatched to one of the servers in a round-robin fashion. + :type topic: str + :param namespace: Identifies a particular interface (i.e. set of methods) + exposed by a server. The default interface has no namespace identifier + and is referred to as the null namespace. + :type namespace: str + :param version: Interfaces have a major.minor version number associated + with them. A minor number increment indicates a backwards compatible + change and an incompatible change is indicated by a major number bump. + Servers may implement multiple major versions and clients may require + indicate that their message requires a particular minimum minor version. + :type version: str + :param server: Clients can request that a message be directed to a specific + server, rather than just one of a pool of servers listening on the topic. + :type server: str + :param fanout: Clients may request that a message be directed to all + servers listening on a topic by setting fanout to ``True``, rather than + just one of them. + :type fanout: bool + """ + + def __init__(self, exchange=None, topic=None, namespace=None, + version=None, server=None, fanout=None): + self.exchange = exchange + self.topic = topic + self.namespace = namespace + self.version = version + self.server = server + self.fanout = fanout + + def __call__(self, **kwargs): + for a in ('exchange', 'topic', 'namespace', + 'version', 'server', 'fanout'): + kwargs.setdefault(a, getattr(self, a)) + return Target(**kwargs) + + def __eq__(self, other): + return vars(self) == vars(other) + + def __ne__(self, other): + return not self == other + + def __repr__(self): + attrs = [] + for a in ['exchange', 'topic', 'namespace', + 'version', 'server', 'fanout']: + v = getattr(self, a) + if v: + attrs.append((a, v)) + values = ', '.join(['%s=%s' % i for i in attrs]) + return '' + + def __hash__(self): + return id(self) diff --git a/oslo_messaging/tests/__init__.py b/oslo_messaging/tests/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/oslo_messaging/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/executors/__init__.py b/oslo_messaging/tests/drivers/__init__.py similarity index 100% rename from tests/executors/__init__.py rename to oslo_messaging/tests/drivers/__init__.py diff --git a/oslo_messaging/tests/drivers/test_impl_qpid.py b/oslo_messaging/tests/drivers/test_impl_qpid.py new file mode 100644 index 000000000..1c90831ba --- /dev/null +++ b/oslo_messaging/tests/drivers/test_impl_qpid.py @@ -0,0 +1,841 @@ +# Copyright (C) 2014 eNovance SAS +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import operator +import random +import threading +import time + +import mock +try: + import qpid +except ImportError: + qpid = None +from six.moves import _thread +import testscenarios +import testtools + +import oslo_messaging +from oslo_messaging._drivers import impl_qpid as qpid_driver +from oslo_messaging.tests import utils as test_utils + + +load_tests = testscenarios.load_tests_apply_scenarios + +QPID_BROKER = 'localhost:5672' + + +class TestQpidDriverLoad(test_utils.BaseTestCase): + + def setUp(self): + super(TestQpidDriverLoad, self).setUp() + self.messaging_conf.transport_driver = 'qpid' + + def test_driver_load(self): + transport = oslo_messaging.get_transport(self.conf) + self.assertIsInstance(transport._driver, qpid_driver.QpidDriver) + + +def _is_qpidd_service_running(): + + """this function checks if the qpid service is running or not.""" + + qpid_running = True + try: + broker = QPID_BROKER + connection = qpid.messaging.Connection(broker) + connection.open() + except Exception: + # qpid service is not running. + qpid_running = False + else: + connection.close() + + return qpid_running + + +class _QpidBaseTestCase(test_utils.BaseTestCase): + + @testtools.skipIf(qpid is None, "qpid not available") + def setUp(self): + super(_QpidBaseTestCase, self).setUp() + self.messaging_conf.transport_driver = 'qpid' + self.fake_qpid = not _is_qpidd_service_running() + + if self.fake_qpid: + self.session_receive = get_fake_qpid_session() + self.session_send = get_fake_qpid_session() + else: + self.broker = QPID_BROKER + # create connection from the qpid.messaging + # connection for the Consumer. + self.con_receive = qpid.messaging.Connection(self.broker) + self.con_receive.open() + # session to receive the messages + self.session_receive = self.con_receive.session() + + # connection for sending the message + self.con_send = qpid.messaging.Connection(self.broker) + self.con_send.open() + # session to send the messages + self.session_send = self.con_send.session() + + # list to store the expected messages and + # the actual received messages + self._expected = [] + self._messages = [] + self.initialized = True + + def tearDown(self): + super(_QpidBaseTestCase, self).tearDown() + + if self.initialized: + if self.fake_qpid: + _fake_session.flush_exchanges() + else: + self.con_receive.close() + self.con_send.close() + + +class TestQpidTransportURL(_QpidBaseTestCase): + + scenarios = [ + ('none', dict(url=None, + expected=[dict(host='localhost:5672', + username='', + password='')])), + ('empty', + dict(url='qpid:///', + expected=[dict(host='localhost:5672', + username='', + password='')])), + ('localhost', + dict(url='qpid://localhost/', + expected=[dict(host='localhost', + username='', + password='')])), + ('no_creds', + dict(url='qpid://host/', + expected=[dict(host='host', + username='', + password='')])), + ('no_port', + dict(url='qpid://user:password@host/', + expected=[dict(host='host', + username='user', + password='password')])), + ('full_url', + dict(url='qpid://user:password@host:10/', + expected=[dict(host='host:10', + username='user', + password='password')])), + ('full_two_url', + dict(url='qpid://user:password@host:10,' + 'user2:password2@host2:12/', + expected=[dict(host='host:10', + username='user', + password='password'), + dict(host='host2:12', + username='user2', + password='password2') + ] + )), + + ] + + @mock.patch.object(qpid_driver.Connection, 'reconnect') + def test_transport_url(self, *args): + transport = oslo_messaging.get_transport(self.conf, self.url) + self.addCleanup(transport.cleanup) + driver = transport._driver + + brokers_params = driver._get_connection().brokers_params + self.assertEqual(sorted(self.expected, + key=operator.itemgetter('host')), + sorted(brokers_params, + key=operator.itemgetter('host'))) + + +class TestQpidInvalidTopologyVersion(_QpidBaseTestCase): + """Unit test cases to test invalid qpid topology version.""" + + scenarios = [ + ('direct', dict(consumer_cls=qpid_driver.DirectConsumer, + consumer_kwargs={}, + publisher_cls=qpid_driver.DirectPublisher, + publisher_kwargs={})), + ('topic', dict(consumer_cls=qpid_driver.TopicConsumer, + consumer_kwargs={'exchange_name': 'openstack'}, + publisher_cls=qpid_driver.TopicPublisher, + publisher_kwargs={'exchange_name': 'openstack'})), + ('fanout', dict(consumer_cls=qpid_driver.FanoutConsumer, + consumer_kwargs={}, + publisher_cls=qpid_driver.FanoutPublisher, + publisher_kwargs={})), + ] + + def setUp(self): + super(TestQpidInvalidTopologyVersion, self).setUp() + self.config(qpid_topology_version=-1) + + def test_invalid_topology_version(self): + def consumer_callback(msg): + pass + + msgid_or_topic = 'test' + + # not using self.assertRaises because + # 1. qpid driver raises Exception(msg) for invalid topology version + # 2. flake8 - H202 assertRaises Exception too broad + exception_msg = ("Invalid value for qpid_topology_version: %d" % + self.conf.qpid_topology_version) + recvd_exc_msg = '' + + try: + self.consumer_cls(self.conf, + self.session_receive, + msgid_or_topic, + consumer_callback, + **self.consumer_kwargs) + except Exception as e: + recvd_exc_msg = e.message + + self.assertEqual(exception_msg, recvd_exc_msg) + + recvd_exc_msg = '' + try: + self.publisher_cls(self.conf, + self.session_send, + topic=msgid_or_topic, + **self.publisher_kwargs) + except Exception as e: + recvd_exc_msg = e.message + + self.assertEqual(exception_msg, recvd_exc_msg) + + +class TestQpidDirectConsumerPublisher(_QpidBaseTestCase): + """Unit test cases to test DirectConsumer and Direct Publisher.""" + + _n_qpid_topology = [ + ('v1', dict(qpid_topology=1)), + ('v2', dict(qpid_topology=2)), + ] + + _n_msgs = [ + ('single', dict(no_msgs=1)), + ('multiple', dict(no_msgs=10)), + ] + + @classmethod + def generate_scenarios(cls): + cls.scenarios = testscenarios.multiply_scenarios(cls._n_qpid_topology, + cls._n_msgs) + + def consumer_callback(self, msg): + # This function will be called by the DirectConsumer + # when any message is received. + # Append the received message into the messages list + # so that the received messages can be validated + # with the expected messages + if isinstance(msg, dict): + self._messages.append(msg['content']) + else: + self._messages.append(msg) + + def test_qpid_direct_consumer_producer(self): + self.msgid = str(random.randint(1, 100)) + + # create a DirectConsumer and DirectPublisher class objects + self.dir_cons = qpid_driver.DirectConsumer(self.conf, + self.session_receive, + self.msgid, + self.consumer_callback) + self.dir_pub = qpid_driver.DirectPublisher(self.conf, + self.session_send, + self.msgid) + + def try_send_msg(no_msgs): + for i in range(no_msgs): + self._expected.append(str(i)) + snd_msg = {'content_type': 'text/plain', 'content': str(i)} + self.dir_pub.send(snd_msg) + + def try_receive_msg(no_msgs): + for i in range(no_msgs): + self.dir_cons.consume() + + thread1 = threading.Thread(target=try_receive_msg, + args=(self.no_msgs,)) + thread2 = threading.Thread(target=try_send_msg, + args=(self.no_msgs,)) + + thread1.start() + thread2.start() + thread1.join() + thread2.join() + + self.assertEqual(self.no_msgs, len(self._messages)) + self.assertEqual(self._expected, self._messages) + + +TestQpidDirectConsumerPublisher.generate_scenarios() + + +class TestQpidTopicAndFanout(_QpidBaseTestCase): + """Unit Test cases to test TopicConsumer and + TopicPublisher classes of the qpid driver + and FanoutConsumer and FanoutPublisher classes + of the qpid driver + """ + + _n_qpid_topology = [ + ('v1', dict(qpid_topology=1)), + ('v2', dict(qpid_topology=2)), + ] + + _n_msgs = [ + ('single', dict(no_msgs=1)), + ('multiple', dict(no_msgs=10)), + ] + + _n_senders = [ + ('single', dict(no_senders=1)), + ('multiple', dict(no_senders=10)), + ] + + _n_receivers = [ + ('single', dict(no_receivers=1)), + ] + _exchange_class = [ + ('topic', dict(consumer_cls=qpid_driver.TopicConsumer, + consumer_kwargs={'exchange_name': 'openstack'}, + publisher_cls=qpid_driver.TopicPublisher, + publisher_kwargs={'exchange_name': 'openstack'}, + topic='topictest.test', + receive_topic='topictest.test')), + ('fanout', dict(consumer_cls=qpid_driver.FanoutConsumer, + consumer_kwargs={}, + publisher_cls=qpid_driver.FanoutPublisher, + publisher_kwargs={}, + topic='fanouttest', + receive_topic='fanouttest')), + ] + + @classmethod + def generate_scenarios(cls): + cls.scenarios = testscenarios.multiply_scenarios(cls._n_qpid_topology, + cls._n_msgs, + cls._n_senders, + cls._n_receivers, + cls._exchange_class) + + def setUp(self): + super(TestQpidTopicAndFanout, self).setUp() + + # to store the expected messages and the + # actual received messages + # + # NOTE(dhellmann): These are dicts, where the base class uses + # lists. + self._expected = {} + self._messages = {} + + self._senders = [] + self._receivers = [] + + self._sender_threads = [] + self._receiver_threads = [] + + def consumer_callback(self, msg): + """callback function called by the ConsumerBase class of + qpid driver. + Message will be received in the format x-y + where x is the sender id and y is the msg number of the sender + extract the sender id 'x' and store the msg 'x-y' with 'x' as + the key + """ + + if isinstance(msg, dict): + msgcontent = msg['content'] + else: + msgcontent = msg + + splitmsg = msgcontent.split('-') + key = _thread.get_ident() + + if key not in self._messages: + self._messages[key] = dict() + + tdict = self._messages[key] + + if splitmsg[0] not in tdict: + tdict[splitmsg[0]] = [] + + tdict[splitmsg[0]].append(msgcontent) + + def _try_send_msg(self, sender_id, no_msgs): + for i in range(no_msgs): + sendmsg = '%s-%s' % (str(sender_id), str(i)) + key = str(sender_id) + # Store the message in the self._expected for each sender. + # This will be used later to + # validate the test by comparing it with the + # received messages by all the receivers + if key not in self._expected: + self._expected[key] = [] + self._expected[key].append(sendmsg) + send_dict = {'content_type': 'text/plain', 'content': sendmsg} + self._senders[sender_id].send(send_dict) + + def _try_receive_msg(self, receiver_id, no_msgs): + for i in range(self.no_senders * no_msgs): + no_of_attempts = 0 + + # ConsumerBase.consume blocks indefinitely until a message + # is received. + # So qpid_receiver.available() is called before calling + # ConsumerBase.consume() so that we are not + # blocked indefinitely + qpid_receiver = self._receivers[receiver_id].get_receiver() + while no_of_attempts < 50: + if qpid_receiver.available() > 0: + self._receivers[receiver_id].consume() + break + no_of_attempts += 1 + time.sleep(0.05) + + def test_qpid_topic_and_fanout(self): + for receiver_id in range(self.no_receivers): + consumer = self.consumer_cls(self.conf, + self.session_receive, + self.receive_topic, + self.consumer_callback, + **self.consumer_kwargs) + self._receivers.append(consumer) + + # create receivers threads + thread = threading.Thread(target=self._try_receive_msg, + args=(receiver_id, self.no_msgs,)) + self._receiver_threads.append(thread) + + for sender_id in range(self.no_senders): + publisher = self.publisher_cls(self.conf, + self.session_send, + topic=self.topic, + **self.publisher_kwargs) + self._senders.append(publisher) + + # create sender threads + thread = threading.Thread(target=self._try_send_msg, + args=(sender_id, self.no_msgs,)) + self._sender_threads.append(thread) + + for thread in self._receiver_threads: + thread.start() + + for thread in self._sender_threads: + thread.start() + + for thread in self._receiver_threads: + thread.join() + + for thread in self._sender_threads: + thread.join() + + # Each receiver should receive all the messages sent by + # the sender(s). + # So, Iterate through each of the receiver items in + # self._messages and compare with the expected messages + # messages. + + self.assertEqual(self.no_senders, len(self._expected)) + self.assertEqual(self.no_receivers, len(self._messages)) + + for key, messages in self._messages.iteritems(): + self.assertEqual(self._expected, messages) + +TestQpidTopicAndFanout.generate_scenarios() + + +class AddressNodeMatcher(object): + def __init__(self, node): + self.node = node + + def __eq__(self, address): + return address.split(';')[0].strip() == self.node + + +class TestDriverInterface(_QpidBaseTestCase): + """Unit Test cases to test the amqpdriver with qpid + """ + + def setUp(self): + super(TestDriverInterface, self).setUp() + self.config(qpid_topology_version=2) + transport = oslo_messaging.get_transport(self.conf) + self.driver = transport._driver + + original_get_connection = self.driver._get_connection + p = mock.patch.object(self.driver, '_get_connection', + side_effect=lambda pooled=True: + original_get_connection(False)) + p.start() + self.addCleanup(p.stop) + + def test_listen_and_direct_send(self): + target = oslo_messaging.Target(exchange="exchange_test", + topic="topic_test", + server="server_test") + + with mock.patch('qpid.messaging.Connection') as conn_cls: + conn = conn_cls.return_value + session = conn.session.return_value + session.receiver.side_effect = [mock.Mock(), mock.Mock(), + mock.Mock()] + + listener = self.driver.listen(target) + listener.conn.direct_send("msg_id", {}) + + self.assertEqual(3, len(listener.conn.consumers)) + + expected_calls = [ + mock.call(AddressNodeMatcher( + 'amq.topic/topic/exchange_test/topic_test')), + mock.call(AddressNodeMatcher( + 'amq.topic/topic/exchange_test/topic_test.server_test')), + mock.call(AddressNodeMatcher('amq.topic/fanout/topic_test')), + ] + session.receiver.assert_has_calls(expected_calls) + session.sender.assert_called_with( + AddressNodeMatcher("amq.direct/msg_id")) + + def test_send(self): + target = oslo_messaging.Target(exchange="exchange_test", + topic="topic_test", + server="server_test") + with mock.patch('qpid.messaging.Connection') as conn_cls: + conn = conn_cls.return_value + session = conn.session.return_value + + self.driver.send(target, {}, {}) + session.sender.assert_called_with(AddressNodeMatcher( + "amq.topic/topic/exchange_test/topic_test.server_test")) + + def test_send_notification(self): + target = oslo_messaging.Target(exchange="exchange_test", + topic="topic_test.info") + with mock.patch('qpid.messaging.Connection') as conn_cls: + conn = conn_cls.return_value + session = conn.session.return_value + + self.driver.send_notification(target, {}, {}, "2.0") + session.sender.assert_called_with(AddressNodeMatcher( + "amq.topic/topic/exchange_test/topic_test.info")) + + +class TestQpidReconnectOrder(test_utils.BaseTestCase): + """Unit Test cases to test reconnection + """ + + @testtools.skipIf(qpid is None, "qpid not available") + def test_reconnect_order(self): + brokers = ['host1', 'host2', 'host3', 'host4', 'host5'] + brokers_count = len(brokers) + + self.config(qpid_hosts=brokers) + + with mock.patch('qpid.messaging.Connection') as conn_mock: + # starting from the first broker in the list + url = oslo_messaging.TransportURL.parse(self.conf, None) + connection = qpid_driver.Connection(self.conf, url) + + # reconnect will advance to the next broker, one broker per + # attempt, and then wrap to the start of the list once the end is + # reached + for _ in range(brokers_count): + connection.reconnect() + + expected = [] + for broker in brokers: + expected.extend([mock.call("%s:5672" % broker), + mock.call().open(), + mock.call().session(), + mock.call().opened(), + mock.call().opened().__nonzero__(), + mock.call().close()]) + + conn_mock.assert_has_calls(expected, any_order=True) + + +def synchronized(func): + func.__lock__ = threading.Lock() + + def synced_func(*args, **kws): + with func.__lock__: + return func(*args, **kws) + + return synced_func + + +class FakeQpidMsgManager(object): + def __init__(self): + self._exchanges = {} + + @synchronized + def add_exchange(self, exchange): + if exchange not in self._exchanges: + self._exchanges[exchange] = {'msgs': [], 'consumers': {}} + + @synchronized + def add_exchange_consumer(self, exchange, consumer_id): + exchange_info = self._exchanges[exchange] + cons_dict = exchange_info['consumers'] + cons_dict[consumer_id] = 0 + + @synchronized + def add_exchange_msg(self, exchange, msg): + exchange_info = self._exchanges[exchange] + exchange_info['msgs'].append(msg) + + def get_exchange_msg(self, exchange, index): + exchange_info = self._exchanges[exchange] + return exchange_info['msgs'][index] + + def get_no_exch_msgs(self, exchange): + exchange_info = self._exchanges[exchange] + return len(exchange_info['msgs']) + + def get_exch_cons_index(self, exchange, consumer_id): + exchange_info = self._exchanges[exchange] + cons_dict = exchange_info['consumers'] + return cons_dict[consumer_id] + + @synchronized + def inc_consumer_index(self, exchange, consumer_id): + exchange_info = self._exchanges[exchange] + cons_dict = exchange_info['consumers'] + cons_dict[consumer_id] += 1 + +_fake_qpid_msg_manager = FakeQpidMsgManager() + + +class FakeQpidSessionSender(object): + def __init__(self, session, id, target, options): + self.session = session + self.id = id + self.target = target + self.options = options + + @synchronized + def send(self, object, sync=True, timeout=None): + _fake_qpid_msg_manager.add_exchange_msg(self.target, object) + + def close(self, timeout=None): + pass + + +class FakeQpidSessionReceiver(object): + + def __init__(self, session, id, source, options): + self.session = session + self.id = id + self.source = source + self.options = options + + @synchronized + def fetch(self, timeout=None): + if timeout is None: + # if timeout is not given, take a default time out + # of 30 seconds to avoid indefinite loop + _timeout = 30 + else: + _timeout = timeout + + deadline = time.time() + _timeout + while time.time() <= deadline: + index = _fake_qpid_msg_manager.get_exch_cons_index(self.source, + self.id) + try: + msg = _fake_qpid_msg_manager.get_exchange_msg(self.source, + index) + except IndexError: + pass + else: + _fake_qpid_msg_manager.inc_consumer_index(self.source, + self.id) + return qpid.messaging.Message(msg) + time.sleep(0.050) + + if timeout is None: + raise Exception('timed out waiting for reply') + + def close(self, timeout=None): + pass + + @synchronized + def available(self): + no_msgs = _fake_qpid_msg_manager.get_no_exch_msgs(self.source) + index = _fake_qpid_msg_manager.get_exch_cons_index(self.source, + self.id) + if no_msgs == 0 or index >= no_msgs: + return 0 + else: + return no_msgs - index + + +class FakeQpidSession(object): + + def __init__(self, connection=None, name=None, transactional=None): + self.connection = connection + self.name = name + self.transactional = transactional + self._receivers = {} + self.conf = None + self.url = None + self._senders = {} + self._sender_id = 0 + self._receiver_id = 0 + + @synchronized + def sender(self, target, **options): + exchange_key = self._extract_exchange_key(target) + _fake_qpid_msg_manager.add_exchange(exchange_key) + + sendobj = FakeQpidSessionSender(self, self._sender_id, + exchange_key, options) + self._senders[self._sender_id] = sendobj + self._sender_id = self._sender_id + 1 + return sendobj + + @synchronized + def receiver(self, source, **options): + exchange_key = self._extract_exchange_key(source) + _fake_qpid_msg_manager.add_exchange(exchange_key) + recvobj = FakeQpidSessionReceiver(self, self._receiver_id, + exchange_key, options) + self._receivers[self._receiver_id] = recvobj + _fake_qpid_msg_manager.add_exchange_consumer(exchange_key, + self._receiver_id) + self._receiver_id += 1 + return recvobj + + def acknowledge(self, message=None, disposition=None, sync=True): + pass + + @synchronized + def flush_exchanges(self): + _fake_qpid_msg_manager._exchanges = {} + + def _extract_exchange_key(self, exchange_msg): + """This function extracts a unique key for the exchange. + This key is used in the dictionary as a 'key' for + this exchange. + Eg. if the exchange_msg (for qpid topology version 1) + is 33/33 ; {"node": {"x-declare": {"auto-delete": true, .... + then 33 is returned as the key. + Eg 2. For topology v2, if the + exchange_msg is - amq.direct/44 ; {"link": {"x-dec....... + then 44 is returned + """ + # first check for ';' + semicolon_split = exchange_msg.split(';') + + # split the first item of semicolon_split with '/' + slash_split = semicolon_split[0].split('/') + # return the last element of the list as the key + key = slash_split[-1] + return key.strip() + + def close(self): + pass + +_fake_session = FakeQpidSession() + + +def get_fake_qpid_session(): + return _fake_session + + +class QPidHATestCase(test_utils.BaseTestCase): + + @testtools.skipIf(qpid is None, "qpid not available") + def setUp(self): + super(QPidHATestCase, self).setUp() + self.brokers = ['host1', 'host2', 'host3', 'host4', 'host5'] + + self.config(qpid_hosts=self.brokers, + qpid_username=None, + qpid_password=None) + + hostname_sets = set() + self.info = {'attempt': 0, + 'fail': False} + + def _connect(myself, broker): + # do as little work that is enough to pass connection attempt + myself.connection = mock.Mock() + hostname = broker['host'] + self.assertNotIn(hostname, hostname_sets) + hostname_sets.add(hostname) + + self.info['attempt'] += 1 + if self.info['fail']: + raise qpid.messaging.exceptions.ConnectionError + + # just make sure connection instantiation does not fail with an + # exception + self.stubs.Set(qpid_driver.Connection, '_connect', _connect) + + # starting from the first broker in the list + url = oslo_messaging.TransportURL.parse(self.conf, None) + self.connection = qpid_driver.Connection(self.conf, url) + self.addCleanup(self.connection.close) + + self.info.update({'attempt': 0, + 'fail': True}) + hostname_sets.clear() + + def test_reconnect_order(self): + self.assertRaises(oslo_messaging.MessageDeliveryFailure, + self.connection.reconnect, + retry=len(self.brokers) - 1) + self.assertEqual(len(self.brokers), self.info['attempt']) + + def test_ensure_four_retries(self): + mock_callback = mock.Mock( + side_effect=qpid.messaging.exceptions.ConnectionError) + self.assertRaises(oslo_messaging.MessageDeliveryFailure, + self.connection.ensure, None, mock_callback, + retry=4) + self.assertEqual(5, self.info['attempt']) + self.assertEqual(1, mock_callback.call_count) + + def test_ensure_one_retry(self): + mock_callback = mock.Mock( + side_effect=qpid.messaging.exceptions.ConnectionError) + self.assertRaises(oslo_messaging.MessageDeliveryFailure, + self.connection.ensure, None, mock_callback, + retry=1) + self.assertEqual(2, self.info['attempt']) + self.assertEqual(1, mock_callback.call_count) + + def test_ensure_no_retry(self): + mock_callback = mock.Mock( + side_effect=qpid.messaging.exceptions.ConnectionError) + self.assertRaises(oslo_messaging.MessageDeliveryFailure, + self.connection.ensure, None, mock_callback, + retry=0) + self.assertEqual(1, self.info['attempt']) + self.assertEqual(1, mock_callback.call_count) diff --git a/oslo_messaging/tests/drivers/test_impl_rabbit.py b/oslo_messaging/tests/drivers/test_impl_rabbit.py new file mode 100644 index 000000000..3aace8b81 --- /dev/null +++ b/oslo_messaging/tests/drivers/test_impl_rabbit.py @@ -0,0 +1,712 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime +import sys +import threading +import time +import uuid + +import fixtures +import kombu +import mock +from oslotest import mockpatch +import testscenarios + +from oslo.config import cfg +from oslo.serialization import jsonutils +import oslo_messaging +from oslo_messaging._drivers import amqpdriver +from oslo_messaging._drivers import common as driver_common +from oslo_messaging._drivers import impl_rabbit as rabbit_driver +from oslo_messaging.tests import utils as test_utils + +load_tests = testscenarios.load_tests_apply_scenarios + + +class TestDeprecatedRabbitDriverLoad(test_utils.BaseTestCase): + + def setUp(self): + super(TestDeprecatedRabbitDriverLoad, self).setUp( + conf=cfg.ConfigOpts()) + self.messaging_conf.transport_driver = 'rabbit' + self.config(fake_rabbit=True) + + def test_driver_load(self): + transport = oslo_messaging.get_transport(self.conf) + self.addCleanup(transport.cleanup) + driver = transport._driver + url = driver._get_connection()._url + + self.assertIsInstance(driver, rabbit_driver.RabbitDriver) + self.assertEqual('memory:////', url) + + +class TestRabbitDriverLoad(test_utils.BaseTestCase): + + scenarios = [ + ('rabbit', dict(transport_driver='rabbit', + url='amqp://guest:guest@localhost:5672//')), + ('kombu', dict(transport_driver='kombu', + url='amqp://guest:guest@localhost:5672//')), + ('rabbit+memory', dict(transport_driver='kombu+memory', + url='memory:///')) + ] + + @mock.patch('oslo_messaging._drivers.impl_rabbit.Connection.ensure') + @mock.patch('oslo_messaging._drivers.impl_rabbit.Connection.reset') + def test_driver_load(self, fake_ensure, fake_reset): + self.messaging_conf.transport_driver = self.transport_driver + transport = oslo_messaging.get_transport(self.conf) + self.addCleanup(transport.cleanup) + driver = transport._driver + url = driver._get_connection()._url + + self.assertIsInstance(driver, rabbit_driver.RabbitDriver) + self.assertEqual(self.url, url) + + +class TestRabbitIterconsume(test_utils.BaseTestCase): + + def test_iterconsume_timeout(self): + transport = oslo_messaging.get_transport(self.conf, + 'kombu+memory:////') + self.addCleanup(transport.cleanup) + deadline = time.time() + 3 + with transport._driver._get_connection() as conn: + conn.iterconsume(timeout=3) + # kombu memory transport doesn't really raise error + # so just simulate a real driver behavior + conn.connection.connection.recoverable_channel_errors = (IOError,) + conn.declare_fanout_consumer("notif.info", lambda msg: True) + with mock.patch('kombu.connection.Connection.drain_events', + side_effect=IOError): + try: + conn.consume(timeout=3) + except driver_common.Timeout: + pass + + self.assertEqual(0, int(deadline - time.time())) + + +class TestRabbitTransportURL(test_utils.BaseTestCase): + + scenarios = [ + ('none', dict(url=None, + expected=["amqp://guest:guest@localhost:5672//"])), + ('memory', dict(url='kombu+memory:////', + expected=["memory:///"])), + ('empty', + dict(url='rabbit:///', + expected=['amqp://guest:guest@localhost:5672/'])), + ('localhost', + dict(url='rabbit://localhost/', + expected=['amqp://:@localhost:5672/'])), + ('virtual_host', + dict(url='rabbit:///vhost', + expected=['amqp://guest:guest@localhost:5672/vhost'])), + ('no_creds', + dict(url='rabbit://host/virtual_host', + expected=['amqp://:@host:5672/virtual_host'])), + ('no_port', + dict(url='rabbit://user:password@host/virtual_host', + expected=['amqp://user:password@host:5672/virtual_host'])), + ('full_url', + dict(url='rabbit://user:password@host:10/virtual_host', + expected=['amqp://user:password@host:10/virtual_host'])), + ('full_two_url', + dict(url='rabbit://user:password@host:10,' + 'user2:password2@host2:12/virtual_host', + expected=["amqp://user:password@host:10/virtual_host", + "amqp://user2:password2@host2:12/virtual_host"] + )), + ] + + def setUp(self): + super(TestRabbitTransportURL, self).setUp() + self.messaging_conf.transport_driver = 'rabbit' + + @mock.patch('oslo_messaging._drivers.impl_rabbit.Connection.ensure') + @mock.patch('oslo_messaging._drivers.impl_rabbit.Connection.reset') + def test_transport_url(self, fake_ensure_connection, fake_reset): + transport = oslo_messaging.get_transport(self.conf, self.url) + self.addCleanup(transport.cleanup) + driver = transport._driver + + urls = driver._get_connection()._url.split(";") + self.assertEqual(sorted(self.expected), sorted(urls)) + + +class TestSendReceive(test_utils.BaseTestCase): + + _n_senders = [ + ('single_sender', dict(n_senders=1)), + ('multiple_senders', dict(n_senders=10)), + ] + + _context = [ + ('empty_context', dict(ctxt={})), + ('with_context', dict(ctxt={'user': 'mark'})), + ] + + _reply = [ + ('rx_id', dict(rx_id=True, reply=None)), + ('none', dict(rx_id=False, reply=None)), + ('empty_list', dict(rx_id=False, reply=[])), + ('empty_dict', dict(rx_id=False, reply={})), + ('false', dict(rx_id=False, reply=False)), + ('zero', dict(rx_id=False, reply=0)), + ] + + _failure = [ + ('success', dict(failure=False)), + ('failure', dict(failure=True, expected=False)), + ('expected_failure', dict(failure=True, expected=True)), + ] + + _timeout = [ + ('no_timeout', dict(timeout=None)), + ('timeout', dict(timeout=0.01)), # FIXME(markmc): timeout=0 is broken? + ] + + @classmethod + def generate_scenarios(cls): + cls.scenarios = testscenarios.multiply_scenarios(cls._n_senders, + cls._context, + cls._reply, + cls._failure, + cls._timeout) + + def test_send_receive(self): + transport = oslo_messaging.get_transport(self.conf, + 'kombu+memory:////') + self.addCleanup(transport.cleanup) + + driver = transport._driver + + target = oslo_messaging.Target(topic='testtopic') + + listener = driver.listen(target) + + senders = [] + replies = [] + msgs = [] + errors = [] + + def stub_error(msg, *a, **kw): + if (a and len(a) == 1 and isinstance(a[0], dict) and a[0]): + a = a[0] + errors.append(str(msg) % a) + + self.stubs.Set(driver_common.LOG, 'error', stub_error) + + def send_and_wait_for_reply(i): + try: + replies.append(driver.send(target, + self.ctxt, + {'tx_id': i}, + wait_for_reply=True, + timeout=self.timeout)) + self.assertFalse(self.failure) + self.assertIsNone(self.timeout) + except (ZeroDivisionError, oslo_messaging.MessagingTimeout) as e: + replies.append(e) + self.assertTrue(self.failure or self.timeout is not None) + + while len(senders) < self.n_senders: + senders.append(threading.Thread(target=send_and_wait_for_reply, + args=(len(senders), ))) + + for i in range(len(senders)): + senders[i].start() + + received = listener.poll() + self.assertIsNotNone(received) + self.assertEqual(self.ctxt, received.ctxt) + self.assertEqual({'tx_id': i}, received.message) + msgs.append(received) + + # reply in reverse, except reply to the first guy second from last + order = list(range(len(senders) - 1, -1, -1)) + if len(order) > 1: + order[-1], order[-2] = order[-2], order[-1] + + for i in order: + if self.timeout is None: + if self.failure: + try: + raise ZeroDivisionError + except Exception: + failure = sys.exc_info() + msgs[i].reply(failure=failure, + log_failure=not self.expected) + elif self.rx_id: + msgs[i].reply({'rx_id': i}) + else: + msgs[i].reply(self.reply) + senders[i].join() + + self.assertEqual(len(senders), len(replies)) + for i, reply in enumerate(replies): + if self.timeout is not None: + self.assertIsInstance(reply, oslo_messaging.MessagingTimeout) + elif self.failure: + self.assertIsInstance(reply, ZeroDivisionError) + elif self.rx_id: + self.assertEqual({'rx_id': order[i]}, reply) + else: + self.assertEqual(self.reply, reply) + + if not self.timeout and self.failure and not self.expected: + self.assertTrue(len(errors) > 0, errors) + else: + self.assertEqual(0, len(errors), errors) + + +TestSendReceive.generate_scenarios() + + +class TestPollAsync(test_utils.BaseTestCase): + + def test_poll_timeout(self): + transport = oslo_messaging.get_transport(self.conf, + 'kombu+memory:////') + self.addCleanup(transport.cleanup) + driver = transport._driver + target = oslo_messaging.Target(topic='testtopic') + listener = driver.listen(target) + received = listener.poll(timeout=0.050) + self.assertIsNone(received) + + +class TestRacyWaitForReply(test_utils.BaseTestCase): + + def test_send_receive(self): + transport = oslo_messaging.get_transport(self.conf, + 'kombu+memory:////') + self.addCleanup(transport.cleanup) + + driver = transport._driver + + target = oslo_messaging.Target(topic='testtopic') + + listener = driver.listen(target) + + senders = [] + replies = [] + msgs = [] + + wait_conditions = [] + orig_reply_waiter = amqpdriver.ReplyWaiter.wait + + def reply_waiter(self, msg_id, timeout): + if wait_conditions: + cond = wait_conditions.pop() + with cond: + cond.notify() + with cond: + cond.wait() + return orig_reply_waiter(self, msg_id, timeout) + + self.stubs.Set(amqpdriver.ReplyWaiter, 'wait', reply_waiter) + + def send_and_wait_for_reply(i, wait_for_reply): + replies.append(driver.send(target, + {}, + {'tx_id': i}, + wait_for_reply=wait_for_reply, + timeout=None)) + + while len(senders) < 2: + t = threading.Thread(target=send_and_wait_for_reply, + args=(len(senders), True)) + t.daemon = True + senders.append(t) + + # test the case then msg_id is not set + t = threading.Thread(target=send_and_wait_for_reply, + args=(len(senders), False)) + t.daemon = True + senders.append(t) + + # Start the first guy, receive his message, but delay his polling + notify_condition = threading.Condition() + wait_conditions.append(notify_condition) + with notify_condition: + senders[0].start() + notify_condition.wait() + + msgs.append(listener.poll()) + self.assertEqual({'tx_id': 0}, msgs[-1].message) + + # Start the second guy, receive his message + senders[1].start() + + msgs.append(listener.poll()) + self.assertEqual({'tx_id': 1}, msgs[-1].message) + + # Reply to both in order, making the second thread queue + # the reply meant for the first thread + msgs[0].reply({'rx_id': 0}) + msgs[1].reply({'rx_id': 1}) + + # Wait for the second thread to finish + senders[1].join() + + # Start the 3rd guy, receive his message + senders[2].start() + + msgs.append(listener.poll()) + self.assertEqual({'tx_id': 2}, msgs[-1].message) + + # Verify the _send_reply was not invoked by driver: + with mock.patch.object(msgs[2], '_send_reply') as method: + msgs[2].reply({'rx_id': 2}) + self.assertEqual(method.call_count, 0) + + # Wait for the 3rd thread to finish + senders[2].join() + + # Let the first thread continue + with notify_condition: + notify_condition.notify() + + # Wait for the first thread to finish + senders[0].join() + + # Verify replies were received out of order + self.assertEqual(len(senders), len(replies)) + self.assertEqual({'rx_id': 1}, replies[0]) + self.assertIsNone(replies[1]) + self.assertEqual({'rx_id': 0}, replies[2]) + + +def _declare_queue(target): + connection = kombu.connection.BrokerConnection(transport='memory') + + # Kludge to speed up tests. + connection.transport.polling_interval = 0.0 + + connection.connect() + channel = connection.channel() + + # work around 'memory' transport bug in 1.1.3 + channel._new_queue('ae.undeliver') + + if target.fanout: + exchange = kombu.entity.Exchange(name=target.topic + '_fanout', + type='fanout', + durable=False, + auto_delete=True) + queue = kombu.entity.Queue(name=target.topic + '_fanout_12345', + channel=channel, + exchange=exchange, + routing_key=target.topic) + if target.server: + exchange = kombu.entity.Exchange(name='openstack', + type='topic', + durable=False, + auto_delete=False) + topic = '%s.%s' % (target.topic, target.server) + queue = kombu.entity.Queue(name=topic, + channel=channel, + exchange=exchange, + routing_key=topic) + else: + exchange = kombu.entity.Exchange(name='openstack', + type='topic', + durable=False, + auto_delete=False) + queue = kombu.entity.Queue(name=target.topic, + channel=channel, + exchange=exchange, + routing_key=target.topic) + + queue.declare() + + return connection, channel, queue + + +class TestRequestWireFormat(test_utils.BaseTestCase): + + _target = [ + ('topic_target', + dict(topic='testtopic', server=None, fanout=False)), + ('server_target', + dict(topic='testtopic', server='testserver', fanout=False)), + # NOTE(markmc): https://github.com/celery/kombu/issues/195 + ('fanout_target', + dict(topic='testtopic', server=None, fanout=True, + skip_msg='Requires kombu>2.5.12 to fix kombu issue #195')), + ] + + _msg = [ + ('empty_msg', + dict(msg={}, expected={})), + ('primitive_msg', + dict(msg={'foo': 'bar'}, expected={'foo': 'bar'})), + ('complex_msg', + dict(msg={'a': {'b': datetime.datetime(1920, 2, 3, 4, 5, 6, 7)}}, + expected={'a': {'b': '1920-02-03T04:05:06.000007'}})), + ] + + _context = [ + ('empty_ctxt', dict(ctxt={}, expected_ctxt={})), + ('user_project_ctxt', + dict(ctxt={'user': 'mark', 'project': 'snarkybunch'}, + expected_ctxt={'_context_user': 'mark', + '_context_project': 'snarkybunch'})), + ] + + @classmethod + def generate_scenarios(cls): + cls.scenarios = testscenarios.multiply_scenarios(cls._msg, + cls._context, + cls._target) + + def setUp(self): + super(TestRequestWireFormat, self).setUp() + self.uuids = [] + self.orig_uuid4 = uuid.uuid4 + self.useFixture(fixtures.MonkeyPatch('uuid.uuid4', self.mock_uuid4)) + + def mock_uuid4(self): + self.uuids.append(self.orig_uuid4()) + return self.uuids[-1] + + def test_request_wire_format(self): + if hasattr(self, 'skip_msg'): + self.skipTest(self.skip_msg) + + transport = oslo_messaging.get_transport(self.conf, + 'kombu+memory:////') + self.addCleanup(transport.cleanup) + + driver = transport._driver + + target = oslo_messaging.Target(topic=self.topic, + server=self.server, + fanout=self.fanout) + + connection, channel, queue = _declare_queue(target) + self.addCleanup(connection.release) + + driver.send(target, self.ctxt, self.msg) + + msgs = [] + + def callback(msg): + msg = channel.message_to_python(msg) + msg.ack() + msgs.append(msg.payload) + + queue.consume(callback=callback, + consumer_tag='1', + nowait=False) + + connection.drain_events() + + self.assertEqual(1, len(msgs)) + self.assertIn('oslo.message', msgs[0]) + + received = msgs[0] + received['oslo.message'] = jsonutils.loads(received['oslo.message']) + + # FIXME(markmc): add _msg_id and _reply_q check + expected_msg = { + '_unique_id': self.uuids[0].hex, + } + expected_msg.update(self.expected) + expected_msg.update(self.expected_ctxt) + + expected = { + 'oslo.version': '2.0', + 'oslo.message': expected_msg, + } + + self.assertEqual(expected, received) + + +TestRequestWireFormat.generate_scenarios() + + +def _create_producer(target): + connection = kombu.connection.BrokerConnection(transport='memory') + + # Kludge to speed up tests. + connection.transport.polling_interval = 0.0 + + connection.connect() + channel = connection.channel() + + # work around 'memory' transport bug in 1.1.3 + channel._new_queue('ae.undeliver') + + if target.fanout: + exchange = kombu.entity.Exchange(name=target.topic + '_fanout', + type='fanout', + durable=False, + auto_delete=True) + producer = kombu.messaging.Producer(exchange=exchange, + channel=channel, + routing_key=target.topic) + elif target.server: + exchange = kombu.entity.Exchange(name='openstack', + type='topic', + durable=False, + auto_delete=False) + topic = '%s.%s' % (target.topic, target.server) + producer = kombu.messaging.Producer(exchange=exchange, + channel=channel, + routing_key=topic) + else: + exchange = kombu.entity.Exchange(name='openstack', + type='topic', + durable=False, + auto_delete=False) + producer = kombu.messaging.Producer(exchange=exchange, + channel=channel, + routing_key=target.topic) + + return connection, producer + + +class TestReplyWireFormat(test_utils.BaseTestCase): + + _target = [ + ('topic_target', + dict(topic='testtopic', server=None, fanout=False)), + ('server_target', + dict(topic='testtopic', server='testserver', fanout=False)), + # NOTE(markmc): https://github.com/celery/kombu/issues/195 + ('fanout_target', + dict(topic='testtopic', server=None, fanout=True, + skip_msg='Requires kombu>2.5.12 to fix kombu issue #195')), + ] + + _msg = [ + ('empty_msg', + dict(msg={}, expected={})), + ('primitive_msg', + dict(msg={'foo': 'bar'}, expected={'foo': 'bar'})), + ('complex_msg', + dict(msg={'a': {'b': '1920-02-03T04:05:06.000007'}}, + expected={'a': {'b': '1920-02-03T04:05:06.000007'}})), + ] + + _context = [ + ('empty_ctxt', dict(ctxt={}, expected_ctxt={})), + ('user_project_ctxt', + dict(ctxt={'_context_user': 'mark', + '_context_project': 'snarkybunch'}, + expected_ctxt={'user': 'mark', 'project': 'snarkybunch'})), + ] + + @classmethod + def generate_scenarios(cls): + cls.scenarios = testscenarios.multiply_scenarios(cls._msg, + cls._context, + cls._target) + + def test_reply_wire_format(self): + if hasattr(self, 'skip_msg'): + self.skipTest(self.skip_msg) + + transport = oslo_messaging.get_transport(self.conf, + 'kombu+memory:////') + self.addCleanup(transport.cleanup) + + driver = transport._driver + + target = oslo_messaging.Target(topic=self.topic, + server=self.server, + fanout=self.fanout) + + listener = driver.listen(target) + + connection, producer = _create_producer(target) + self.addCleanup(connection.release) + + msg = { + 'oslo.version': '2.0', + 'oslo.message': {} + } + + msg['oslo.message'].update(self.msg) + msg['oslo.message'].update(self.ctxt) + + msg['oslo.message'].update({ + '_msg_id': uuid.uuid4().hex, + '_unique_id': uuid.uuid4().hex, + '_reply_q': 'reply_' + uuid.uuid4().hex, + }) + + msg['oslo.message'] = jsonutils.dumps(msg['oslo.message']) + + producer.publish(msg) + + received = listener.poll() + self.assertIsNotNone(received) + self.assertEqual(self.expected_ctxt, received.ctxt) + self.assertEqual(self.expected, received.message) + + +TestReplyWireFormat.generate_scenarios() + + +class RpcKombuHATestCase(test_utils.BaseTestCase): + + def setUp(self): + super(RpcKombuHATestCase, self).setUp() + self.brokers = ['host1', 'host2', 'host3', 'host4', 'host5'] + self.config(rabbit_hosts=self.brokers, + rabbit_retry_interval=0.01, + rabbit_retry_backoff=0.01, + kombu_reconnect_delay=0) + + self.kombu_connect = mock.Mock() + self.useFixture(mockpatch.Patch( + 'kombu.connection.Connection.connect', + side_effect=self.kombu_connect)) + self.useFixture(mockpatch.Patch( + 'kombu.connection.Connection.channel')) + + # starting from the first broker in the list + url = oslo_messaging.TransportURL.parse(self.conf, None) + self.connection = rabbit_driver.Connection(self.conf, url) + self.addCleanup(self.connection.close) + + def test_ensure_four_retry(self): + mock_callback = mock.Mock(side_effect=IOError) + self.assertRaises(oslo_messaging.MessageDeliveryFailure, + self.connection.ensure, None, mock_callback, + retry=4) + self.assertEqual(5, self.kombu_connect.call_count) + self.assertEqual(6, mock_callback.call_count) + + def test_ensure_one_retry(self): + mock_callback = mock.Mock(side_effect=IOError) + self.assertRaises(oslo_messaging.MessageDeliveryFailure, + self.connection.ensure, None, mock_callback, + retry=1) + self.assertEqual(2, self.kombu_connect.call_count) + self.assertEqual(3, mock_callback.call_count) + + def test_ensure_no_retry(self): + mock_callback = mock.Mock(side_effect=IOError) + self.assertRaises(oslo_messaging.MessageDeliveryFailure, + self.connection.ensure, None, mock_callback, + retry=0) + self.assertEqual(1, self.kombu_connect.call_count) + self.assertEqual(2, mock_callback.call_count) diff --git a/oslo_messaging/tests/drivers/test_impl_zmq.py b/oslo_messaging/tests/drivers/test_impl_zmq.py new file mode 100644 index 000000000..1d3d4089d --- /dev/null +++ b/oslo_messaging/tests/drivers/test_impl_zmq.py @@ -0,0 +1,504 @@ +# Copyright 2014 Canonical, Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import socket + +import fixtures +import mock +import testtools + +from oslo.utils import importutils +import oslo_messaging +from oslo_messaging.tests import utils as test_utils + +# NOTE(jamespage) the zmq driver implementation is currently tied +# to eventlet so we have to monkey_patch to support testing +# eventlet is not yet py3 compatible, so skip if not installed +eventlet = importutils.try_import('eventlet') +if eventlet: + eventlet.monkey_patch() + +impl_zmq = importutils.try_import('oslo_messaging._drivers.impl_zmq') + +LOG = logging.getLogger(__name__) + + +def get_unused_port(): + """Returns an unused port on localhost.""" + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(('localhost', 0)) + port = s.getsockname()[1] + s.close() + return port + + +class TestConfZmqDriverLoad(test_utils.BaseTestCase): + + @testtools.skipIf(impl_zmq is None, "zmq not available") + def setUp(self): + super(TestConfZmqDriverLoad, self).setUp() + self.messaging_conf.transport_driver = 'zmq' + + def test_driver_load(self): + transport = oslo_messaging.get_transport(self.conf) + self.assertIsInstance(transport._driver, impl_zmq.ZmqDriver) + + +class stopRpc(object): + def __init__(self, attrs): + self.attrs = attrs + + def __call__(self): + if self.attrs['reactor']: + self.attrs['reactor'].close() + if self.attrs['driver']: + self.attrs['driver'].cleanup() + + +class TestZmqBasics(test_utils.BaseTestCase): + + @testtools.skipIf(impl_zmq is None, "zmq not available") + def setUp(self): + super(TestZmqBasics, self).setUp() + self.messaging_conf.transport_driver = 'zmq' + # Get driver + transport = oslo_messaging.get_transport(self.conf) + self.driver = transport._driver + + # Set config values + self.internal_ipc_dir = self.useFixture(fixtures.TempDir()).path + kwargs = {'rpc_zmq_bind_address': '127.0.0.1', + 'rpc_zmq_host': '127.0.0.1', + 'rpc_response_timeout': 5, + 'rpc_zmq_port': get_unused_port(), + 'rpc_zmq_ipc_dir': self.internal_ipc_dir} + self.config(**kwargs) + + # Start RPC + LOG.info("Running internal zmq receiver.") + self.reactor = impl_zmq.ZmqProxy(self.conf) + self.reactor.consume_in_thread() + + self.matchmaker = impl_zmq._get_matchmaker(host='127.0.0.1') + self.addCleanup(stopRpc(self.__dict__)) + + def test_start_stop_listener(self): + target = oslo_messaging.Target(topic='testtopic') + listener = self.driver.listen(target) + result = listener.poll(0.01) + self.assertEqual(result, None) + + def test_send_receive_raises(self): + """Call() without method.""" + target = oslo_messaging.Target(topic='testtopic') + self.driver.listen(target) + self.assertRaises( + KeyError, + self.driver.send, + target, {}, {'tx_id': 1}, wait_for_reply=True) + + @mock.patch('oslo_messaging._drivers.impl_zmq.ZmqIncomingMessage') + def test_send_receive_topic(self, mock_msg): + """Call() with method.""" + mock_msg.return_value = msg = mock.MagicMock() + msg.received = received = mock.MagicMock() + received.failure = False + received.reply = True + msg.condition = condition = mock.MagicMock() + condition.wait.return_value = True + + target = oslo_messaging.Target(topic='testtopic') + self.driver.listen(target) + result = self.driver.send( + target, {}, + {'method': 'hello-world', 'tx_id': 1}, + wait_for_reply=True) + self.assertEqual(result, True) + + @mock.patch('oslo_messaging._drivers.impl_zmq._call', autospec=True) + def test_send_receive_fanout(self, mock_call): + target = oslo_messaging.Target(topic='testtopic', fanout=True) + self.driver.listen(target) + + mock_call.__name__ = '_call' + mock_call.return_value = [True] + + result = self.driver.send( + target, {}, + {'method': 'hello-world', 'tx_id': 1}, + wait_for_reply=True) + + self.assertEqual(result, True) + mock_call.assert_called_once_with( + 'tcp://127.0.0.1:%s' % self.conf['rpc_zmq_port'], + {}, 'fanout~testtopic.127.0.0.1', + {'tx_id': 1, 'method': 'hello-world'}, + None, False, []) + + @mock.patch('oslo_messaging._drivers.impl_zmq._call', autospec=True) + def test_send_receive_direct(self, mock_call): + # Also verifies fix for bug http://pad.lv/1301723 + target = oslo_messaging.Target(topic='testtopic', server='localhost') + self.driver.listen(target) + + mock_call.__name__ = '_call' + mock_call.return_value = [True] + + result = self.driver.send( + target, {}, + {'method': 'hello-world', 'tx_id': 1}, + wait_for_reply=True) + + self.assertEqual(result, True) + mock_call.assert_called_once_with( + 'tcp://localhost:%s' % self.conf['rpc_zmq_port'], + {}, 'testtopic.localhost', + {'tx_id': 1, 'method': 'hello-world'}, + None, False, []) + + +class TestZmqSocket(test_utils.BaseTestCase): + + @testtools.skipIf(impl_zmq is None, "zmq not available") + def setUp(self): + super(TestZmqSocket, self).setUp() + self.messaging_conf.transport_driver = 'zmq' + # Get driver + transport = oslo_messaging.get_transport(self.conf) + self.driver = transport._driver + + @mock.patch('oslo_messaging._drivers.impl_zmq.ZmqSocket.subscribe') + @mock.patch('oslo_messaging._drivers.impl_zmq.zmq.Context') + def test_zmqsocket_init_type_pull(self, mock_context, mock_subscribe): + mock_ctxt = mock.Mock() + mock_context.return_value = mock_ctxt + mock_sock = mock.Mock() + mock_ctxt.socket = mock.Mock(return_value=mock_sock) + mock_sock.connect = mock.Mock() + mock_sock.bind = mock.Mock() + addr = '127.0.0.1' + + sock = impl_zmq.ZmqSocket(addr, impl_zmq.zmq.PULL, bind=False, + subscribe=None) + self.assertTrue(sock.can_recv) + self.assertFalse(sock.can_send) + self.assertFalse(sock.can_sub) + self.assertTrue(mock_sock.connect.called) + self.assertFalse(mock_sock.bind.called) + + @mock.patch('oslo_messaging._drivers.impl_zmq.ZmqSocket.subscribe') + @mock.patch('oslo_messaging._drivers.impl_zmq.zmq.Context') + def test_zmqsocket_init_type_sub(self, mock_context, mock_subscribe): + mock_ctxt = mock.Mock() + mock_context.return_value = mock_ctxt + mock_sock = mock.Mock() + mock_ctxt.socket = mock.Mock(return_value=mock_sock) + mock_sock.connect = mock.Mock() + mock_sock.bind = mock.Mock() + addr = '127.0.0.1' + + sock = impl_zmq.ZmqSocket(addr, impl_zmq.zmq.SUB, bind=False, + subscribe=None) + self.assertTrue(sock.can_recv) + self.assertFalse(sock.can_send) + self.assertTrue(sock.can_sub) + self.assertTrue(mock_sock.connect.called) + self.assertFalse(mock_sock.bind.called) + + @mock.patch('oslo_messaging._drivers.impl_zmq.ZmqSocket.subscribe') + @mock.patch('oslo_messaging._drivers.impl_zmq.zmq.Context') + def test_zmqsocket_init_type_push(self, mock_context, mock_subscribe): + mock_ctxt = mock.Mock() + mock_context.return_value = mock_ctxt + mock_sock = mock.Mock() + mock_ctxt.socket = mock.Mock(return_value=mock_sock) + mock_sock.connect = mock.Mock() + mock_sock.bind = mock.Mock() + addr = '127.0.0.1' + + sock = impl_zmq.ZmqSocket(addr, impl_zmq.zmq.PUSH, bind=False, + subscribe=None) + self.assertFalse(sock.can_recv) + self.assertTrue(sock.can_send) + self.assertFalse(sock.can_sub) + self.assertTrue(mock_sock.connect.called) + self.assertFalse(mock_sock.bind.called) + + @mock.patch('oslo_messaging._drivers.impl_zmq.ZmqSocket.subscribe') + @mock.patch('oslo_messaging._drivers.impl_zmq.zmq.Context') + def test_zmqsocket_init_type_pub(self, mock_context, mock_subscribe): + mock_ctxt = mock.Mock() + mock_context.return_value = mock_ctxt + mock_sock = mock.Mock() + mock_ctxt.socket = mock.Mock(return_value=mock_sock) + mock_sock.connect = mock.Mock() + mock_sock.bind = mock.Mock() + addr = '127.0.0.1' + + sock = impl_zmq.ZmqSocket(addr, impl_zmq.zmq.PUB, bind=False, + subscribe=None) + self.assertFalse(sock.can_recv) + self.assertTrue(sock.can_send) + self.assertFalse(sock.can_sub) + self.assertTrue(mock_sock.connect.called) + self.assertFalse(mock_sock.bind.called) + + +class TestZmqIncomingMessage(test_utils.BaseTestCase): + + @testtools.skipIf(impl_zmq is None, "zmq not available") + def setUp(self): + super(TestZmqIncomingMessage, self).setUp() + self.messaging_conf.transport_driver = 'zmq' + # Get driver + transport = oslo_messaging.get_transport(self.conf) + self.driver = transport._driver + + def test_zmqincomingmessage(self): + msg = impl_zmq.ZmqIncomingMessage(mock.Mock(), None, 'msg.foo') + msg.reply("abc") + self.assertIsInstance( + msg.received, impl_zmq.ZmqIncomingMessage.ReceivedReply) + self.assertIsInstance( + msg.received, impl_zmq.ZmqIncomingMessage.ReceivedReply) + self.assertEqual(msg.received.reply, "abc") + msg.requeue() + + +class TestZmqConnection(test_utils.BaseTestCase): + + @testtools.skipIf(impl_zmq is None, "zmq not available") + def setUp(self): + super(TestZmqConnection, self).setUp() + self.messaging_conf.transport_driver = 'zmq' + # Get driver + transport = oslo_messaging.get_transport(self.conf) + self.driver = transport._driver + + # Set config values + self.internal_ipc_dir = self.useFixture(fixtures.TempDir()).path + kwargs = {'rpc_zmq_bind_address': '127.0.0.1', + 'rpc_zmq_host': '127.0.0.1', + 'rpc_response_timeout': 5, + 'rpc_zmq_port': get_unused_port(), + 'rpc_zmq_ipc_dir': self.internal_ipc_dir} + self.config(**kwargs) + + # Start RPC + LOG.info("Running internal zmq receiver.") + self.reactor = impl_zmq.ZmqProxy(self.conf) + self.reactor.consume_in_thread() + + self.matchmaker = impl_zmq._get_matchmaker(host='127.0.0.1') + self.addCleanup(stopRpc(self.__dict__)) + + @mock.patch('oslo_messaging._drivers.impl_zmq.ZmqReactor', autospec=True) + def test_zmqconnection_create_consumer(self, mock_reactor): + + mock_reactor.register = mock.Mock() + conn = impl_zmq.Connection(self.driver) + topic = 'topic.foo' + context = mock.Mock() + inaddr = ('ipc://%s/zmq_topic_topic.127.0.0.1' % + (self.internal_ipc_dir)) + # No Fanout + conn.create_consumer(topic, context) + conn.reactor.register.assert_called_with(context, inaddr, + impl_zmq.zmq.PULL, + subscribe=None, in_bind=False) + + # Reset for next bunch of checks + conn.reactor.register.reset_mock() + + # Fanout + inaddr = ('ipc://%s/zmq_topic_fanout~topic' % + (self.internal_ipc_dir)) + conn.create_consumer(topic, context, fanout='subscriber.foo') + conn.reactor.register.assert_called_with(context, inaddr, + impl_zmq.zmq.SUB, + subscribe='subscriber.foo', + in_bind=False) + + @mock.patch('oslo_messaging._drivers.impl_zmq.ZmqReactor', autospec=True) + def test_zmqconnection_create_consumer_topic_exists(self, mock_reactor): + mock_reactor.register = mock.Mock() + conn = impl_zmq.Connection(self.driver) + topic = 'topic.foo' + context = mock.Mock() + inaddr = ('ipc://%s/zmq_topic_topic.127.0.0.1' % + (self.internal_ipc_dir)) + + conn.create_consumer(topic, context) + conn.reactor.register.assert_called_with( + context, inaddr, impl_zmq.zmq.PULL, subscribe=None, in_bind=False) + conn.reactor.register.reset_mock() + # Call again with same topic + conn.create_consumer(topic, context) + self.assertFalse(conn.reactor.register.called) + + @mock.patch('oslo_messaging._drivers.impl_zmq._get_matchmaker', + autospec=True) + @mock.patch('oslo_messaging._drivers.impl_zmq.ZmqReactor', autospec=True) + def test_zmqconnection_close(self, mock_reactor, mock_getmatchmaker): + conn = impl_zmq.Connection(self.driver) + conn.reactor.close = mock.Mock() + mock_getmatchmaker.return_value.stop_heartbeat = mock.Mock() + conn.close() + self.assertTrue(mock_getmatchmaker.return_value.stop_heartbeat.called) + self.assertTrue(conn.reactor.close.called) + + @mock.patch('oslo_messaging._drivers.impl_zmq.ZmqReactor', autospec=True) + def test_zmqconnection_wait(self, mock_reactor): + conn = impl_zmq.Connection(self.driver) + conn.reactor.wait = mock.Mock() + conn.wait() + self.assertTrue(conn.reactor.wait.called) + + @mock.patch('oslo_messaging._drivers.impl_zmq._get_matchmaker', + autospec=True) + @mock.patch('oslo_messaging._drivers.impl_zmq.ZmqReactor', autospec=True) + def test_zmqconnection_consume_in_thread(self, mock_reactor, + mock_getmatchmaker): + mock_getmatchmaker.return_value.start_heartbeat = mock.Mock() + conn = impl_zmq.Connection(self.driver) + conn.reactor.consume_in_thread = mock.Mock() + conn.consume_in_thread() + self.assertTrue(mock_getmatchmaker.return_value.start_heartbeat.called) + self.assertTrue(conn.reactor.consume_in_thread.called) + + +class TestZmqListener(test_utils.BaseTestCase): + + @testtools.skipIf(impl_zmq is None, "zmq not available") + def setUp(self): + super(TestZmqListener, self).setUp() + self.messaging_conf.transport_driver = 'zmq' + # Get driver + transport = oslo_messaging.get_transport(self.conf) + self.driver = transport._driver + + # Set config values + self.internal_ipc_dir = self.useFixture(fixtures.TempDir()).path + kwargs = {'rpc_zmq_bind_address': '127.0.0.1', + 'rpc_zmq_host': '127.0.0.1', + 'rpc_response_timeout': 5, + 'rpc_zmq_port': get_unused_port(), + 'rpc_zmq_ipc_dir': self.internal_ipc_dir} + self.config(**kwargs) + + # Start RPC + LOG.info("Running internal zmq receiver.") + self.reactor = impl_zmq.ZmqProxy(self.conf) + self.reactor.consume_in_thread() + + self.matchmaker = impl_zmq._get_matchmaker(host='127.0.0.1') + self.addCleanup(stopRpc(self.__dict__)) + + def test_zmqlistener_no_msg(self): + listener = impl_zmq.ZmqListener(self.driver) + # Timeout = 0 should return straight away since the queue is empty + listener.poll(timeout=0) + + def test_zmqlistener_w_msg(self): + listener = impl_zmq.ZmqListener(self.driver) + kwargs = {'a': 1, 'b': 2} + m = mock.Mock() + ctxt = mock.Mock(autospec=impl_zmq.RpcContext) + eventlet.spawn_n(listener.dispatch, ctxt, 0, + m.fake_method, 'name.space', **kwargs) + resp = listener.poll(timeout=10) + msg = {'method': m.fake_method, 'namespace': 'name.space', + 'args': kwargs} + self.assertEqual(resp.message, msg) + + +class TestZmqDriver(test_utils.BaseTestCase): + + @testtools.skipIf(impl_zmq is None, "zmq not available") + def setUp(self): + super(TestZmqDriver, self).setUp() + self.messaging_conf.transport_driver = 'zmq' + # Get driver + transport = oslo_messaging.get_transport(self.conf) + self.driver = transport._driver + + # Set config values + self.internal_ipc_dir = self.useFixture(fixtures.TempDir()).path + kwargs = {'rpc_zmq_bind_address': '127.0.0.1', + 'rpc_zmq_host': '127.0.0.1', + 'rpc_response_timeout': 5, + 'rpc_zmq_port': get_unused_port(), + 'rpc_zmq_ipc_dir': self.internal_ipc_dir} + self.config(**kwargs) + + # Start RPC + LOG.info("Running internal zmq receiver.") + self.reactor = impl_zmq.ZmqProxy(self.conf) + self.reactor.consume_in_thread() + + self.matchmaker = impl_zmq._get_matchmaker(host='127.0.0.1') + self.addCleanup(stopRpc(self.__dict__)) + + @mock.patch('oslo_messaging._drivers.impl_zmq._cast', autospec=True) + @mock.patch('oslo_messaging._drivers.impl_zmq._multi_send', autospec=True) + def test_zmqdriver_send(self, mock_multi_send, mock_cast): + context = mock.Mock(autospec=impl_zmq.RpcContext) + topic = 'testtopic' + msg = 'jeronimo' + self.driver.send(oslo_messaging.Target(topic=topic), context, msg, + False, 0, False) + mock_multi_send.assert_called_with(mock_cast, context, topic, msg, + allowed_remote_exmods=[], + envelope=False) + + @mock.patch('oslo_messaging._drivers.impl_zmq._cast', autospec=True) + @mock.patch('oslo_messaging._drivers.impl_zmq._multi_send', autospec=True) + def test_zmqdriver_send_notification(self, mock_multi_send, mock_cast): + context = mock.Mock(autospec=impl_zmq.RpcContext) + topic = 'testtopic.foo' + topic_reformat = 'testtopic-foo' + msg = 'jeronimo' + self.driver.send_notification(oslo_messaging.Target(topic=topic), + context, msg, False, False) + mock_multi_send.assert_called_with(mock_cast, context, topic_reformat, + msg, allowed_remote_exmods=[], + envelope=False) + + @mock.patch('oslo_messaging._drivers.impl_zmq.ZmqListener', autospec=True) + @mock.patch('oslo_messaging._drivers.impl_zmq.Connection', autospec=True) + def test_zmqdriver_listen(self, mock_connection, mock_listener): + mock_listener.return_value = listener = mock.Mock() + mock_connection.return_value = conn = mock.Mock() + conn.create_consumer = mock.Mock() + conn.consume_in_thread = mock.Mock() + topic = 'testtopic.foo' + self.driver.listen(oslo_messaging.Target(topic=topic)) + conn.create_consumer.assert_called_with(topic, listener, fanout=True) + + @mock.patch('oslo_messaging._drivers.impl_zmq.ZmqListener', autospec=True) + @mock.patch('oslo_messaging._drivers.impl_zmq.Connection', autospec=True) + def test_zmqdriver_listen_for_notification(self, mock_connection, + mock_listener): + mock_listener.return_value = listener = mock.Mock() + mock_connection.return_value = conn = mock.Mock() + conn.create_consumer = mock.Mock() + conn.consume_in_thread = mock.Mock() + topic = 'testtopic.foo' + data = [(oslo_messaging.Target(topic=topic), 0)] + # NOTE(jamespage): Pooling not supported, just pass None for now. + self.driver.listen_for_notifications(data, None) + conn.create_consumer.assert_called_with("%s-%s" % (topic, 0), listener) diff --git a/oslo_messaging/tests/drivers/test_matchmaker.py b/oslo_messaging/tests/drivers/test_matchmaker.py new file mode 100644 index 000000000..40608f4f5 --- /dev/null +++ b/oslo_messaging/tests/drivers/test_matchmaker.py @@ -0,0 +1,69 @@ +# Copyright 2014 Canonical, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import testtools + +from oslo.utils import importutils +from oslo_messaging.tests import utils as test_utils + +# NOTE(jamespage) matchmaker tied directly to eventlet +# which is not yet py3 compatible - skip if import fails +matchmaker = ( + importutils.try_import('oslo_messaging._drivers.matchmaker')) + + +@testtools.skipIf(not matchmaker, "matchmaker/eventlet unavailable") +class MatchmakerTest(test_utils.BaseTestCase): + + def test_fanout_binding(self): + matcher = matchmaker.MatchMakerBase() + matcher.add_binding( + matchmaker.FanoutBinding(), matchmaker.DirectExchange()) + self.assertEqual(matcher.queues('hello.world'), []) + self.assertEqual( + matcher.queues('fanout~fantasy.unicorn'), + [('fanout~fantasy.unicorn', 'unicorn')]) + self.assertEqual( + matcher.queues('fanout~fantasy.pony'), + [('fanout~fantasy.pony', 'pony')]) + + def test_topic_binding(self): + matcher = matchmaker.MatchMakerBase() + matcher.add_binding( + matchmaker.TopicBinding(), matchmaker.StubExchange()) + self.assertEqual( + matcher.queues('hello-world'), [('hello-world', None)]) + + def test_direct_binding(self): + matcher = matchmaker.MatchMakerBase() + matcher.add_binding( + matchmaker.DirectBinding(), matchmaker.StubExchange()) + self.assertEqual( + matcher.queues('hello.server'), [('hello.server', None)]) + self.assertEqual(matcher.queues('hello-world'), []) + + def test_localhost_match(self): + matcher = matchmaker.MatchMakerLocalhost() + self.assertEqual( + matcher.queues('hello.server'), [('hello.server', 'server')]) + + # Gets remapped due to localhost exchange + # all bindings default to first match. + self.assertEqual( + matcher.queues('fanout~testing.server'), + [('fanout~testing.localhost', 'localhost')]) + + self.assertEqual( + matcher.queues('hello-world'), + [('hello-world.localhost', 'localhost')]) diff --git a/oslo_messaging/tests/drivers/test_matchmaker_redis.py b/oslo_messaging/tests/drivers/test_matchmaker_redis.py new file mode 100644 index 000000000..19f6bc148 --- /dev/null +++ b/oslo_messaging/tests/drivers/test_matchmaker_redis.py @@ -0,0 +1,78 @@ +# Copyright 2014 Canonical, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import testtools + +from oslo.utils import importutils +from oslo_messaging.tests import utils as test_utils + +redis = importutils.try_import('redis') +matchmaker_redis = ( + importutils.try_import('oslo_messaging._drivers.matchmaker_redis')) + + +def redis_available(): + '''Helper to see if local redis server is running''' + if not redis: + return False + try: + c = redis.StrictRedis(socket_timeout=1) + c.ping() + return True + except redis.exceptions.ConnectionError: + return False + + +@testtools.skipIf(not matchmaker_redis, "matchmaker/eventlet unavailable") +@testtools.skipIf(not redis_available(), "redis unavailable") +class RedisMatchMakerTest(test_utils.BaseTestCase): + + def setUp(self): + super(RedisMatchMakerTest, self).setUp() + self.ring_data = { + "conductor": ["controller1", "node1", "node2", "node3"], + "scheduler": ["controller1", "node1", "node2", "node3"], + "network": ["controller1", "node1", "node2", "node3"], + "cert": ["controller1"], + "console": ["controller1"], + "consoleauth": ["controller1"]} + self.matcher = matchmaker_redis.MatchMakerRedis() + self.populate() + + def tearDown(self): + super(RedisMatchMakerTest, self).tearDown() + c = redis.StrictRedis() + c.flushdb() + + def populate(self): + for k, hosts in self.ring_data.items(): + for h in hosts: + self.matcher.register(k, h) + + def test_direct(self): + self.assertEqual( + self.matcher.queues('cert.controller1'), + [('cert.controller1', 'controller1')]) + + def test_register(self): + self.matcher.register('cert', 'keymaster') + self.assertEqual( + sorted(self.matcher.redis.smembers('cert')), + ['cert.controller1', 'cert.keymaster']) + + def test_unregister(self): + self.matcher.unregister('conductor', 'controller1') + self.assertEqual( + sorted(self.matcher.redis.smembers('conductor')), + ['conductor.node1', 'conductor.node2', 'conductor.node3']) diff --git a/oslo_messaging/tests/drivers/test_matchmaker_ring.py b/oslo_messaging/tests/drivers/test_matchmaker_ring.py new file mode 100644 index 000000000..48c5a65cc --- /dev/null +++ b/oslo_messaging/tests/drivers/test_matchmaker_ring.py @@ -0,0 +1,73 @@ +# Copyright 2014 Canonical, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import testtools + +from oslo.utils import importutils +from oslo_messaging.tests import utils as test_utils + +# NOTE(jamespage) matchmaker tied directly to eventlet +# which is not yet py3 compatible - skip if import fails +matchmaker_ring = ( + importutils.try_import('oslo_messaging._drivers.matchmaker_ring')) + + +@testtools.skipIf(not matchmaker_ring, "matchmaker/eventlet unavailable") +class MatchmakerRingTest(test_utils.BaseTestCase): + + def setUp(self): + super(MatchmakerRingTest, self).setUp() + self.ring_data = { + "conductor": ["controller1", "node1", "node2", "node3"], + "scheduler": ["controller1", "node1", "node2", "node3"], + "network": ["controller1", "node1", "node2", "node3"], + "cert": ["controller1"], + "console": ["controller1"], + "consoleauth": ["controller1"]} + self.matcher = matchmaker_ring.MatchMakerRing(self.ring_data) + + def test_direct(self): + self.assertEqual( + self.matcher.queues('cert.controller1'), + [('cert.controller1', 'controller1')]) + self.assertEqual( + self.matcher.queues('conductor.node1'), + [('conductor.node1', 'node1')]) + + def test_fanout(self): + self.assertEqual( + self.matcher.queues('fanout~conductor'), + [('fanout~conductor.controller1', 'controller1'), + ('fanout~conductor.node1', 'node1'), + ('fanout~conductor.node2', 'node2'), + ('fanout~conductor.node3', 'node3')]) + + def test_bare_topic(self): + # Round robins through the hosts on the topic + self.assertEqual( + self.matcher.queues('scheduler'), + [('scheduler.controller1', 'controller1')]) + self.assertEqual( + self.matcher.queues('scheduler'), + [('scheduler.node1', 'node1')]) + self.assertEqual( + self.matcher.queues('scheduler'), + [('scheduler.node2', 'node2')]) + self.assertEqual( + self.matcher.queues('scheduler'), + [('scheduler.node3', 'node3')]) + # Cycles loop + self.assertEqual( + self.matcher.queues('scheduler'), + [('scheduler.controller1', 'controller1')]) diff --git a/oslo_messaging/tests/drivers/test_pool.py b/oslo_messaging/tests/drivers/test_pool.py new file mode 100644 index 000000000..a0b6ab47c --- /dev/null +++ b/oslo_messaging/tests/drivers/test_pool.py @@ -0,0 +1,124 @@ + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import threading +import uuid + +import testscenarios + +from oslo_messaging._drivers import pool +from oslo_messaging.tests import utils as test_utils + +load_tests = testscenarios.load_tests_apply_scenarios + + +class PoolTestCase(test_utils.BaseTestCase): + + _max_size = [ + ('default_size', dict(max_size=None, n_iters=4)), + ('set_max_size', dict(max_size=10, n_iters=10)), + ] + + _create_error = [ + ('no_create_error', dict(create_error=False)), + ('create_error', dict(create_error=True)), + ] + + @classmethod + def generate_scenarios(cls): + cls.scenarios = testscenarios.multiply_scenarios(cls._max_size, + cls._create_error) + + class TestPool(pool.Pool): + + def create(self): + return uuid.uuid4() + + class ThreadWaitWaiter(object): + + """A gross hack. + + Stub out the condition variable's wait() method and spin until it + has been called by each thread. + """ + + def __init__(self, cond, n_threads, stubs): + self.cond = cond + self.stubs = stubs + self.n_threads = n_threads + self.n_waits = 0 + self.orig_wait = cond.wait + + def count_waits(**kwargs): + self.n_waits += 1 + self.orig_wait(**kwargs) + self.stubs.Set(self.cond, 'wait', count_waits) + + def wait(self): + while self.n_waits < self.n_threads: + pass + self.stubs.Set(self.cond, 'wait', self.orig_wait) + + def test_pool(self): + kwargs = {} + if self.max_size is not None: + kwargs['max_size'] = self.max_size + + p = self.TestPool(**kwargs) + + if self.create_error: + def create_error(): + raise RuntimeError + orig_create = p.create + self.stubs.Set(p, 'create', create_error) + self.assertRaises(RuntimeError, p.get) + self.stubs.Set(p, 'create', orig_create) + + objs = [] + for i in range(self.n_iters): + objs.append(p.get()) + self.assertIsInstance(objs[i], uuid.UUID) + + def wait_for_obj(): + o = p.get() + self.assertIn(o, objs) + + waiter = self.ThreadWaitWaiter(p._cond, self.n_iters, self.stubs) + + threads = [] + for i in range(self.n_iters): + t = threading.Thread(target=wait_for_obj) + t.start() + threads.append(t) + + waiter.wait() + + for o in objs: + p.put(o) + + for t in threads: + t.join() + + for o in objs: + p.put(o) + + for o in p.iter_free(): + self.assertIn(o, objs) + objs.remove(o) + + self.assertEqual([], objs) + + +PoolTestCase.generate_scenarios() diff --git a/oslo_messaging/tests/executors/__init__.py b/oslo_messaging/tests/executors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/executors/test_executor.py b/oslo_messaging/tests/executors/test_executor.py similarity index 95% rename from tests/executors/test_executor.py rename to oslo_messaging/tests/executors/test_executor.py index 6e0376680..cb321ddef 100644 --- a/tests/executors/test_executor.py +++ b/oslo_messaging/tests/executors/test_executor.py @@ -25,13 +25,13 @@ import mock import testscenarios import testtools -from oslo.messaging._executors import impl_blocking +from oslo_messaging._executors import impl_blocking try: - from oslo.messaging._executors import impl_eventlet + from oslo_messaging._executors import impl_eventlet except ImportError: impl_eventlet = None -from oslo.messaging._executors import impl_thread -from tests import utils as test_utils +from oslo_messaging._executors import impl_thread +from oslo_messaging.tests import utils as test_utils load_tests = testscenarios.load_tests_apply_scenarios diff --git a/oslo_messaging/tests/functional/__init__.py b/oslo_messaging/tests/functional/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/oslo_messaging/tests/functional/test_functional.py b/oslo_messaging/tests/functional/test_functional.py new file mode 100644 index 000000000..e1ac025b8 --- /dev/null +++ b/oslo_messaging/tests/functional/test_functional.py @@ -0,0 +1,284 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import oslo_messaging + +from testtools import matchers + +from oslo_messaging.tests.functional import utils + + +class CallTestCase(utils.SkipIfNoTransportURL): + def test_specific_server(self): + group = self.useFixture(utils.RpcServerGroupFixture(self.url)) + client = group.client(1) + client.append(text='open') + self.assertEqual('openstack', client.append(text='stack')) + client.add(increment=2) + self.assertEqual(12, client.add(increment=10)) + self.assertEqual(9, client.subtract(increment=3)) + self.assertEqual('openstack', group.servers[1].endpoint.sval) + self.assertEqual(9, group.servers[1].endpoint.ival) + for i in [0, 2]: + self.assertEqual('', group.servers[i].endpoint.sval) + self.assertEqual(0, group.servers[i].endpoint.ival) + + def test_server_in_group(self): + group = self.useFixture(utils.RpcServerGroupFixture(self.url)) + + client = group.client() + data = [c for c in 'abcdefghijklmn'] + for i in data: + client.append(text=i) + + for s in group.servers: + self.assertThat(len(s.endpoint.sval), matchers.GreaterThan(0)) + actual = [[c for c in s.endpoint.sval] for s in group.servers] + self.assertThat(actual, utils.IsValidDistributionOf(data)) + + def test_different_exchanges(self): + t = self.useFixture(utils.TransportFixture(self.url)) + # If the different exchanges are not honoured, then the + # teardown may hang unless we broadcast all control messages + # to each server + group1 = self.useFixture( + utils.RpcServerGroupFixture(self.url, transport=t, + use_fanout_ctrl=True)) + group2 = self.useFixture( + utils.RpcServerGroupFixture(self.url, exchange="a", + transport=t, + use_fanout_ctrl=True)) + group3 = self.useFixture( + utils.RpcServerGroupFixture(self.url, exchange="b", + transport=t, + use_fanout_ctrl=True)) + + client1 = group1.client(1) + data1 = [c for c in 'abcdefghijklmn'] + for i in data1: + client1.append(text=i) + + client2 = group2.client() + data2 = [c for c in 'opqrstuvwxyz'] + for i in data2: + client2.append(text=i) + + actual1 = [[c for c in s.endpoint.sval] for s in group1.servers] + self.assertThat(actual1, utils.IsValidDistributionOf(data1)) + actual1 = [c for c in group1.servers[1].endpoint.sval] + self.assertThat([actual1], utils.IsValidDistributionOf(data1)) + for s in group1.servers: + expected = len(data1) if group1.servers.index(s) == 1 else 0 + self.assertEqual(expected, len(s.endpoint.sval)) + self.assertEqual(0, s.endpoint.ival) + + actual2 = [[c for c in s.endpoint.sval] for s in group2.servers] + for s in group2.servers: + self.assertThat(len(s.endpoint.sval), matchers.GreaterThan(0)) + self.assertEqual(0, s.endpoint.ival) + self.assertThat(actual2, utils.IsValidDistributionOf(data2)) + + for s in group3.servers: + self.assertEqual(0, len(s.endpoint.sval)) + self.assertEqual(0, s.endpoint.ival) + + def test_timeout(self): + transport = self.useFixture(utils.TransportFixture(self.url)) + target = oslo_messaging.Target(topic="no_such_topic") + c = utils.ClientStub(transport.transport, target, timeout=1) + self.assertThat(c.ping, + matchers.raises(oslo_messaging.MessagingTimeout)) + + def test_exception(self): + group = self.useFixture(utils.RpcServerGroupFixture(self.url)) + client = group.client(1) + client.add(increment=2) + f = lambda: client.subtract(increment=3) + self.assertThat(f, matchers.raises(ValueError)) + + +class CastTestCase(utils.SkipIfNoTransportURL): + # Note: casts return immediately, so these tests utilise a special + # internal sync() cast to ensure prior casts are complete before + # making the necessary assertions. + + def test_specific_server(self): + group = self.useFixture(utils.RpcServerGroupFixture(self.url)) + client = group.client(1, cast=True) + client.append(text='open') + client.append(text='stack') + client.add(increment=2) + client.add(increment=10) + group.sync() + + self.assertEqual('openstack', group.servers[1].endpoint.sval) + self.assertEqual(12, group.servers[1].endpoint.ival) + for i in [0, 2]: + self.assertEqual('', group.servers[i].endpoint.sval) + self.assertEqual(0, group.servers[i].endpoint.ival) + + def test_server_in_group(self): + group = self.useFixture(utils.RpcServerGroupFixture(self.url)) + client = group.client(cast=True) + for i in range(20): + client.add(increment=1) + group.sync() + total = 0 + for s in group.servers: + ival = s.endpoint.ival + self.assertThat(ival, matchers.GreaterThan(0)) + self.assertThat(ival, matchers.LessThan(20)) + total += ival + self.assertEqual(20, total) + + def test_fanout(self): + group = self.useFixture(utils.RpcServerGroupFixture(self.url)) + client = group.client('all', cast=True) + client.append(text='open') + client.append(text='stack') + client.add(increment=2) + client.add(increment=10) + group.sync(server='all') + for s in group.servers: + self.assertEqual('openstack', s.endpoint.sval) + self.assertEqual(12, s.endpoint.ival) + + +class NotifyTestCase(utils.SkipIfNoTransportURL): + # NOTE(sileht): Each test must not use the same topics + # to be run in parallel + + def test_simple(self): + transport = self.useFixture(utils.TransportFixture(self.url)) + listener = self.useFixture( + utils.NotificationFixture(transport.transport, + ['test_simple'])) + transport.wait() + notifier = listener.notifier('abc') + + notifier.info({}, 'test', 'Hello World!') + event = listener.events.get(timeout=1) + self.assertEqual('info', event[0]) + self.assertEqual('test', event[1]) + self.assertEqual('Hello World!', event[2]) + self.assertEqual('abc', event[3]) + + def test_multiple_topics(self): + transport = self.useFixture(utils.TransportFixture(self.url)) + listener = self.useFixture( + utils.NotificationFixture(transport.transport, + ['a', 'b'])) + transport.wait() + a = listener.notifier('pub-a', topic='a') + b = listener.notifier('pub-b', topic='b') + + sent = { + 'pub-a': [a, 'test-a', 'payload-a'], + 'pub-b': [b, 'test-b', 'payload-b'] + } + for e in sent.values(): + e[0].info({}, e[1], e[2]) + + received = {} + while len(received) < len(sent): + e = listener.events.get(timeout=1) + received[e[3]] = e + + for key in received: + actual = received[key] + expected = sent[key] + self.assertEqual('info', actual[0]) + self.assertEqual(expected[1], actual[1]) + self.assertEqual(expected[2], actual[2]) + + def test_multiple_servers(self): + transport = self.useFixture(utils.TransportFixture(self.url)) + listener_a = self.useFixture( + utils.NotificationFixture(transport.transport, + ['test-topic'])) + listener_b = self.useFixture( + utils.NotificationFixture(transport.transport, + ['test-topic'])) + transport.wait() + n = listener_a.notifier('pub') + + events_out = [('test-%s' % c, 'payload-%s' % c) for c in 'abcdefgh'] + + for event_type, payload in events_out: + n.info({}, event_type, payload) + + events_in = [[(e[1], e[2]) for e in listener_a.get_events()], + [(e[1], e[2]) for e in listener_b.get_events()]] + + self.assertThat(events_in, utils.IsValidDistributionOf(events_out)) + for stream in events_in: + self.assertThat(len(stream), matchers.GreaterThan(0)) + + def test_independent_topics(self): + transport = self.useFixture(utils.TransportFixture(self.url)) + listener_a = self.useFixture( + utils.NotificationFixture(transport.transport, + ['1'])) + listener_b = self.useFixture( + utils.NotificationFixture(transport.transport, + ['2'])) + transport.wait() + + a = listener_a.notifier('pub-1', topic='1') + b = listener_b.notifier('pub-2', topic='2') + + a_out = [('test-1-%s' % c, 'payload-1-%s' % c) for c in 'abcdefgh'] + for event_type, payload in a_out: + a.info({}, event_type, payload) + + b_out = [('test-2-%s' % c, 'payload-2-%s' % c) for c in 'ijklmnop'] + for event_type, payload in b_out: + b.info({}, event_type, payload) + + for expected in a_out: + actual = listener_a.events.get(timeout=0.5) + self.assertEqual('info', actual[0]) + self.assertEqual(expected[0], actual[1]) + self.assertEqual(expected[1], actual[2]) + self.assertEqual('pub-1', actual[3]) + + for expected in b_out: + actual = listener_b.events.get(timeout=0.5) + self.assertEqual('info', actual[0]) + self.assertEqual(expected[0], actual[1]) + self.assertEqual(expected[1], actual[2]) + self.assertEqual('pub-2', actual[3]) + + def test_all_categories(self): + transport = self.useFixture(utils.TransportFixture(self.url)) + listener = self.useFixture(utils.NotificationFixture( + transport.transport, ['test_all_categories'])) + transport.wait() + n = listener.notifier('abc') + + cats = ['debug', 'audit', 'info', 'warn', 'error', 'critical'] + events = [(getattr(n, c), c, 'type-' + c, c + '-data') for c in cats] + for e in events: + e[0]({}, e[2], e[3]) + + # order between events with different categories is not guaranteed + received = {} + for expected in events: + e = listener.events.get(timeout=0.5) + received[e[0]] = e + + for expected in events: + actual = received[expected[1]] + self.assertEqual(expected[1], actual[0]) + self.assertEqual(expected[2], actual[1]) + self.assertEqual(expected[3], actual[2]) diff --git a/oslo_messaging/tests/functional/utils.py b/oslo_messaging/tests/functional/utils.py new file mode 100644 index 000000000..bbdd11b78 --- /dev/null +++ b/oslo_messaging/tests/functional/utils.py @@ -0,0 +1,344 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import threading +import time +import uuid + +import fixtures +from six import moves + +from oslo.config import cfg +import oslo_messaging +from oslo_messaging.notify import notifier +from oslo_messaging.tests import utils as test_utils + + +class TestServerEndpoint(object): + """This MessagingServer that will be used during functional testing.""" + + def __init__(self): + self.ival = 0 + self.sval = '' + + def add(self, ctxt, increment): + self.ival += increment + return self.ival + + def subtract(self, ctxt, increment): + if self.ival < increment: + raise ValueError("ival can't go negative!") + self.ival -= increment + return self.ival + + def append(self, ctxt, text): + self.sval += text + return self.sval + + +class TransportFixture(fixtures.Fixture): + """Fixture defined to setup the oslo_messaging transport.""" + + def __init__(self, url): + self.url = url + + def setUp(self): + super(TransportFixture, self).setUp() + self.transport = oslo_messaging.get_transport(cfg.CONF, url=self.url) + + def cleanUp(self): + self.transport.cleanup() + super(TransportFixture, self).cleanUp() + + def wait(self): + if self.url.startswith("rabbit") or self.url.startswith("qpid"): + time.sleep(0.5) + + +class RpcServerFixture(fixtures.Fixture): + """Fixture to setup the TestServerEndpoint.""" + + def __init__(self, transport, target, endpoint=None, ctrl_target=None): + super(RpcServerFixture, self).__init__() + self.transport = transport + self.target = target + self.endpoint = endpoint or TestServerEndpoint() + self.syncq = moves.queue.Queue() + self.ctrl_target = ctrl_target or self.target + + def setUp(self): + super(RpcServerFixture, self).setUp() + endpoints = [self.endpoint, self] + self.server = oslo_messaging.get_rpc_server(self.transport, + self.target, + endpoints) + self._ctrl = oslo_messaging.RPCClient(self.transport, self.ctrl_target) + self._start() + + def cleanUp(self): + self._stop() + super(RpcServerFixture, self).cleanUp() + + def _start(self): + self.thread = threading.Thread(target=self.server.start) + self.thread.daemon = True + self.thread.start() + + def _stop(self): + self.server.stop() + self._ctrl.cast({}, 'ping') + self.server.wait() + self.thread.join() + + def ping(self, ctxt): + pass + + def sync(self, ctxt, item): + self.syncq.put(item) + + +class RpcServerGroupFixture(fixtures.Fixture): + def __init__(self, url, topic=None, names=None, exchange=None, + transport=None, use_fanout_ctrl=False): + self.url = url + # NOTE(sileht): topic and servier_name must be uniq + # to be able to run all tests in parallel + self.topic = topic or str(uuid.uuid4()) + self.names = names or ["server_%i_%s" % (i, uuid.uuid4()) + for i in range(3)] + self.exchange = exchange + self.targets = [self._target(server=n) for n in self.names] + self.transport = transport + self.use_fanout_ctrl = use_fanout_ctrl + + def setUp(self): + super(RpcServerGroupFixture, self).setUp() + if not self.transport: + self.transport = self.useFixture(TransportFixture(self.url)) + self.servers = [self.useFixture(self._server(t)) for t in self.targets] + self.transport.wait() + + def _target(self, server=None, fanout=False): + t = oslo_messaging.Target(exchange=self.exchange, topic=self.topic) + t.server = server + t.fanout = fanout + return t + + def _server(self, target): + ctrl = None + if self.use_fanout_ctrl: + ctrl = self._target(fanout=True) + return RpcServerFixture(self.transport.transport, target, + ctrl_target=ctrl) + + def client(self, server=None, cast=False): + if server: + if server == 'all': + target = self._target(fanout=True) + elif server >= 0 and server < len(self.targets): + target = self.targets[server] + else: + raise ValueError("Invalid value for server: %r" % server) + else: + target = self._target() + return ClientStub(self.transport.transport, target, cast=cast, + timeout=5) + + def sync(self, server=None): + if server: + if server == 'all': + c = self.client(server='all', cast=True) + c.sync(item='x') + for s in self.servers: + s.syncq.get(timeout=5) + elif server >= 0 and server < len(self.targets): + c = self.client(server=server, cast=True) + c.sync(item='x') + self.servers[server].syncq.get(timeout=5) + else: + raise ValueError("Invalid value for server: %r" % server) + else: + for i in range(len(self.servers)): + self.client(i).ping() + + +class RpcCall(object): + def __init__(self, client, method, context): + self.client = client + self.method = method + self.context = context + + def __call__(self, **kwargs): + self.context['time'] = time.ctime() + self.context['cast'] = False + result = self.client.call(self.context, self.method, **kwargs) + return result + + +class RpcCast(RpcCall): + def __call__(self, **kwargs): + self.context['time'] = time.ctime() + self.context['cast'] = True + self.client.cast(self.context, self.method, **kwargs) + + +class ClientStub(object): + def __init__(self, transport, target, cast=False, name=None, **kwargs): + self.name = name or "functional-tests" + self.cast = cast + self.client = oslo_messaging.RPCClient(transport, target, **kwargs) + + def __getattr__(self, name): + context = {"application": self.name} + if self.cast: + return RpcCast(self.client, name, context) + else: + return RpcCall(self.client, name, context) + + +class InvalidDistribution(object): + def __init__(self, original, received): + self.original = original + self.received = received + self.missing = [] + self.extra = [] + self.wrong_order = [] + + def describe(self): + text = "Sent %s, got %s; " % (self.original, self.received) + e1 = ["%r was missing" % m for m in self.missing] + e2 = ["%r was not expected" % m for m in self.extra] + e3 = ["%r expected before %r" % (m[0], m[1]) for m in self.wrong_order] + return text + ", ".join(e1 + e2 + e3) + + def __len__(self): + return len(self.extra) + len(self.missing) + len(self.wrong_order) + + def get_details(self): + return {} + + +class IsValidDistributionOf(object): + """Test whether a given list can be split into particular + sub-lists. All items in the original list must be in exactly one + sub-list, and must appear in that sub-list in the same order with + respect to any other items as in the original list. + """ + def __init__(self, original): + self.original = original + + def __str__(self): + return 'IsValidDistribution(%s)' % self.original + + def match(self, actual): + errors = InvalidDistribution(self.original, actual) + received = [[i for i in l] for l in actual] + + def _remove(obj, lists): + for l in lists: + if obj in l: + front = l[0] + l.remove(obj) + return front + return None + + for item in self.original: + o = _remove(item, received) + if not o: + errors.missing += item + elif item != o: + errors.wrong_order.append([item, o]) + for l in received: + errors.extra += l + return errors or None + + +class SkipIfNoTransportURL(test_utils.BaseTestCase): + def setUp(self): + super(SkipIfNoTransportURL, self).setUp() + self.url = os.environ.get('TRANSPORT_URL') + if not self.url: + self.skipTest("No transport url configured") + + +class NotificationFixture(fixtures.Fixture): + def __init__(self, transport, topics): + super(NotificationFixture, self).__init__() + self.transport = transport + self.topics = topics + self.events = moves.queue.Queue() + self.name = str(id(self)) + + def setUp(self): + super(NotificationFixture, self).setUp() + targets = [oslo_messaging.Target(topic=t) for t in self.topics] + # add a special topic for internal notifications + targets.append(oslo_messaging.Target(topic=self.name)) + self.server = oslo_messaging.get_notification_listener( + self.transport, + targets, + [self]) + self._ctrl = self.notifier('internal', topic=self.name) + self._start() + + def cleanUp(self): + self._stop() + super(NotificationFixture, self).cleanUp() + + def _start(self): + self.thread = threading.Thread(target=self.server.start) + self.thread.daemon = True + self.thread.start() + + def _stop(self): + self.server.stop() + self._ctrl.sample({}, 'shutdown', 'shutdown') + self.server.wait() + self.thread.join() + + def notifier(self, publisher, topic=None): + return notifier.Notifier(self.transport, + publisher, + driver='messaging', + topic=topic or self.topics[0]) + + def debug(self, ctxt, publisher, event_type, payload, metadata): + self.events.put(['debug', event_type, payload, publisher]) + + def audit(self, ctxt, publisher, event_type, payload, metadata): + self.events.put(['audit', event_type, payload, publisher]) + + def info(self, ctxt, publisher, event_type, payload, metadata): + self.events.put(['info', event_type, payload, publisher]) + + def warn(self, ctxt, publisher, event_type, payload, metadata): + self.events.put(['warn', event_type, payload, publisher]) + + def error(self, ctxt, publisher, event_type, payload, metadata): + self.events.put(['error', event_type, payload, publisher]) + + def critical(self, ctxt, publisher, event_type, payload, metadata): + self.events.put(['critical', event_type, payload, publisher]) + + def sample(self, ctxt, publisher, event_type, payload, metadata): + pass # Just used for internal shutdown control + + def get_events(self, timeout=0.5): + results = [] + try: + while True: + results.append(self.events.get(timeout=timeout)) + except moves.queue.Empty: + pass + return results diff --git a/oslo_messaging/tests/notify/__init__.py b/oslo_messaging/tests/notify/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/oslo_messaging/tests/notify/test_dispatcher.py b/oslo_messaging/tests/notify/test_dispatcher.py new file mode 100644 index 000000000..029f737ec --- /dev/null +++ b/oslo_messaging/tests/notify/test_dispatcher.py @@ -0,0 +1,149 @@ + +# Copyright 2013 eNovance +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import itertools + +import mock +import testscenarios + +from oslo.utils import timeutils +import oslo_messaging +from oslo_messaging.notify import dispatcher as notify_dispatcher +from oslo_messaging.tests import utils as test_utils + +load_tests = testscenarios.load_tests_apply_scenarios + + +notification_msg = dict( + publisher_id="publisher_id", + event_type="compute.start", + payload={"info": "fuu"}, + message_id="uuid", + timestamp=str(timeutils.utcnow()) +) + + +class TestDispatcher(test_utils.BaseTestCase): + + scenarios = [ + ('no_endpoints', + dict(endpoints=[], + endpoints_expect_calls=[], + priority='info', + ex=None, + return_value=oslo_messaging.NotificationResult.HANDLED)), + ('one_endpoints', + dict(endpoints=[['warn']], + endpoints_expect_calls=['warn'], + priority='warn', + ex=None, + return_value=oslo_messaging.NotificationResult.HANDLED)), + ('two_endpoints_only_one_match', + dict(endpoints=[['warn'], ['info']], + endpoints_expect_calls=[None, 'info'], + priority='info', + ex=None, + return_value=oslo_messaging.NotificationResult.HANDLED)), + ('two_endpoints_both_match', + dict(endpoints=[['debug', 'info'], ['info', 'debug']], + endpoints_expect_calls=['debug', 'debug'], + priority='debug', + ex=None, + return_value=oslo_messaging.NotificationResult.HANDLED)), + ('no_return_value', + dict(endpoints=[['warn']], + endpoints_expect_calls=['warn'], + priority='warn', + ex=None, return_value=None)), + ('requeue', + dict(endpoints=[['debug', 'warn']], + endpoints_expect_calls=['debug'], + priority='debug', msg=notification_msg, + ex=None, + return_value=oslo_messaging.NotificationResult.REQUEUE)), + ('exception', + dict(endpoints=[['debug', 'warn']], + endpoints_expect_calls=['debug'], + priority='debug', msg=notification_msg, + ex=Exception, + return_value=oslo_messaging.NotificationResult.HANDLED)), + ] + + def test_dispatcher(self): + endpoints = [] + for endpoint_methods in self.endpoints: + e = mock.Mock(spec=endpoint_methods) + endpoints.append(e) + for m in endpoint_methods: + method = getattr(e, m) + if self.ex: + method.side_effect = self.ex() + else: + method.return_value = self.return_value + + msg = notification_msg.copy() + msg['priority'] = self.priority + + targets = [oslo_messaging.Target(topic='notifications')] + dispatcher = notify_dispatcher.NotificationDispatcher( + targets, endpoints, None, allow_requeue=True, pool=None) + + # check it listen on wanted topics + self.assertEqual(sorted(set((targets[0], prio) + for prio in itertools.chain.from_iterable( + self.endpoints))), + sorted(dispatcher._targets_priorities)) + + incoming = mock.Mock(ctxt={}, message=msg) + with dispatcher(incoming) as callback: + callback() + + # check endpoint callbacks are called or not + for i, endpoint_methods in enumerate(self.endpoints): + for m in endpoint_methods: + if m == self.endpoints_expect_calls[i]: + method = getattr(endpoints[i], m) + method.assert_called_once_with( + {}, + msg['publisher_id'], + msg['event_type'], + msg['payload'], { + 'timestamp': mock.ANY, + 'message_id': mock.ANY + }) + else: + self.assertEqual(0, endpoints[i].call_count) + + if self.ex: + self.assertEqual(1, incoming.acknowledge.call_count) + self.assertEqual(0, incoming.requeue.call_count) + elif self.return_value == oslo_messaging.NotificationResult.HANDLED \ + or self.return_value is None: + self.assertEqual(1, incoming.acknowledge.call_count) + self.assertEqual(0, incoming.requeue.call_count) + elif self.return_value == oslo_messaging.NotificationResult.REQUEUE: + self.assertEqual(0, incoming.acknowledge.call_count) + self.assertEqual(1, incoming.requeue.call_count) + + @mock.patch('oslo_messaging.notify.dispatcher.LOG') + def test_dispatcher_unknown_prio(self, mylog): + msg = notification_msg.copy() + msg['priority'] = 'what???' + dispatcher = notify_dispatcher.NotificationDispatcher( + [mock.Mock()], [mock.Mock()], None, allow_requeue=True, pool=None) + with dispatcher(mock.Mock(ctxt={}, message=msg)) as callback: + callback() + mylog.warning.assert_called_once_with('Unknown priority "%s"', + 'what???') diff --git a/oslo_messaging/tests/notify/test_listener.py b/oslo_messaging/tests/notify/test_listener.py new file mode 100644 index 000000000..e1aa510b8 --- /dev/null +++ b/oslo_messaging/tests/notify/test_listener.py @@ -0,0 +1,398 @@ + +# Copyright 2013 eNovance +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import threading + +import mock +import testscenarios + +from oslo.config import cfg +import oslo_messaging +from oslo_messaging.notify import dispatcher +from oslo_messaging.tests import utils as test_utils + +load_tests = testscenarios.load_tests_apply_scenarios + + +class RestartableListenerThread(object): + def __init__(self, listener): + self.listener = listener + self.thread = None + + def start(self): + if self.thread is None: + self.thread = threading.Thread(target=self.listener.start) + self.thread.daemon = True + self.thread.start() + + def stop(self): + if self.thread is not None: + self.listener.stop() + self.listener.wait() + self.thread.join() + self.thread = None + + def wait_end(self): + self.thread.join(timeout=15) + return self.thread.isAlive() + + +class ListenerSetupMixin(object): + + class ListenerTracker(object): + def __init__(self, expect_messages): + self._expect_messages = expect_messages + self._received_msgs = 0 + self.listeners = [] + + def info(self, ctxt, publisher_id, event_type, payload, metadata): + self._received_msgs += 1 + if self._expect_messages == self._received_msgs: + self.stop() + + def wait_for(self, expect_messages): + while expect_messages != self._received_msgs: + yield + + def stop(self): + for listener in self.listeners: + # Check start() does nothing with a running listener + listener.start() + listener.stop() + listener.wait() + self.listeners = [] + + def setUp(self): + self.trackers = {} + self.addCleanup(self._stop_trackers) + + def _stop_trackers(self): + for pool in self.trackers: + self.trackers[pool].stop() + self.trackers = {} + + def _setup_listener(self, transport, endpoints, expect_messages, + targets=None, pool=None): + + if pool is None: + tracker_name = '__default__' + else: + tracker_name = pool + + if targets is None: + targets = [oslo_messaging.Target(topic='testtopic')] + + tracker = self.trackers.setdefault( + tracker_name, self.ListenerTracker(expect_messages)) + listener = oslo_messaging.get_notification_listener( + transport, targets=targets, endpoints=[tracker] + endpoints, + allow_requeue=True, pool=pool) + tracker.listeners.append(listener) + + thread = RestartableListenerThread(listener) + thread.start() + return thread + + def _setup_notifier(self, transport, topic='testtopic', + publisher_id='testpublisher'): + return oslo_messaging.Notifier(transport, topic=topic, + driver='messaging', + publisher_id=publisher_id) + + +class TestNotifyListener(test_utils.BaseTestCase, ListenerSetupMixin): + + def __init__(self, *args): + super(TestNotifyListener, self).__init__(*args) + ListenerSetupMixin.__init__(self) + + def setUp(self): + super(TestNotifyListener, self).setUp(conf=cfg.ConfigOpts()) + ListenerSetupMixin.setUp(self) + + def test_constructor(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + target = oslo_messaging.Target(topic='foo') + endpoints = [object()] + + listener = oslo_messaging.get_notification_listener( + transport, [target], endpoints) + + self.assertIs(listener.conf, self.conf) + self.assertIs(listener.transport, transport) + self.assertIsInstance(listener.dispatcher, + dispatcher.NotificationDispatcher) + self.assertIs(listener.dispatcher.endpoints, endpoints) + self.assertEqual('blocking', listener.executor) + + def test_no_target_topic(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + listener = oslo_messaging.get_notification_listener( + transport, + [oslo_messaging.Target()], + [mock.Mock()]) + try: + listener.start() + except Exception as ex: + self.assertIsInstance(ex, oslo_messaging.InvalidTarget, ex) + else: + self.assertTrue(False) + + def test_unknown_executor(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + try: + oslo_messaging.get_notification_listener(transport, [], [], + executor='foo') + except Exception as ex: + self.assertIsInstance(ex, oslo_messaging.ExecutorLoadFailure) + self.assertEqual('foo', ex.executor) + else: + self.assertTrue(False) + + def test_one_topic(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + endpoint = mock.Mock() + endpoint.info.return_value = None + listener_thread = self._setup_listener(transport, [endpoint], 1) + + notifier = self._setup_notifier(transport) + notifier.info({}, 'an_event.start', 'test message') + + self.assertFalse(listener_thread.wait_end()) + + endpoint.info.assert_called_once_with( + {}, 'testpublisher', 'an_event.start', 'test message', + {'message_id': mock.ANY, 'timestamp': mock.ANY}) + + def test_two_topics(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + endpoint = mock.Mock() + endpoint.info.return_value = None + targets = [oslo_messaging.Target(topic="topic1"), + oslo_messaging.Target(topic="topic2")] + listener_thread = self._setup_listener(transport, [endpoint], 2, + targets=targets) + notifier = self._setup_notifier(transport, topic='topic1') + notifier.info({'ctxt': '1'}, 'an_event.start1', 'test') + notifier = self._setup_notifier(transport, topic='topic2') + notifier.info({'ctxt': '2'}, 'an_event.start2', 'test') + + self.assertFalse(listener_thread.wait_end()) + + endpoint.info.assert_has_calls([ + mock.call({'ctxt': '1'}, 'testpublisher', + 'an_event.start1', 'test', + {'timestamp': mock.ANY, 'message_id': mock.ANY}), + mock.call({'ctxt': '2'}, 'testpublisher', + 'an_event.start2', 'test', + {'timestamp': mock.ANY, 'message_id': mock.ANY})], + any_order=True) + + def test_two_exchanges(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + endpoint = mock.Mock() + endpoint.info.return_value = None + targets = [oslo_messaging.Target(topic="topic", + exchange="exchange1"), + oslo_messaging.Target(topic="topic", + exchange="exchange2")] + listener_thread = self._setup_listener(transport, [endpoint], 2, + targets=targets) + + notifier = self._setup_notifier(transport, topic="topic") + + def mock_notifier_exchange(name): + def side_effect(target, ctxt, message, version, retry): + target.exchange = name + return transport._driver.send_notification(target, ctxt, + message, version, + retry=retry) + transport._send_notification = mock.MagicMock( + side_effect=side_effect) + + notifier.info({'ctxt': '0'}, + 'an_event.start', 'test message default exchange') + mock_notifier_exchange('exchange1') + notifier.info({'ctxt': '1'}, + 'an_event.start', 'test message exchange1') + mock_notifier_exchange('exchange2') + notifier.info({'ctxt': '2'}, + 'an_event.start', 'test message exchange2') + + self.assertFalse(listener_thread.wait_end()) + + endpoint.info.assert_has_calls([ + mock.call({'ctxt': '1'}, 'testpublisher', 'an_event.start', + 'test message exchange1', + {'timestamp': mock.ANY, 'message_id': mock.ANY}), + mock.call({'ctxt': '2'}, 'testpublisher', 'an_event.start', + 'test message exchange2', + {'timestamp': mock.ANY, 'message_id': mock.ANY})], + any_order=True) + + def test_two_endpoints(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + endpoint1 = mock.Mock() + endpoint1.info.return_value = None + endpoint2 = mock.Mock() + endpoint2.info.return_value = oslo_messaging.NotificationResult.HANDLED + listener_thread = self._setup_listener(transport, + [endpoint1, endpoint2], 1) + notifier = self._setup_notifier(transport) + notifier.info({}, 'an_event.start', 'test') + + self.assertFalse(listener_thread.wait_end()) + + endpoint1.info.assert_called_once_with( + {}, 'testpublisher', 'an_event.start', 'test', { + 'timestamp': mock.ANY, + 'message_id': mock.ANY}) + + endpoint2.info.assert_called_once_with( + {}, 'testpublisher', 'an_event.start', 'test', { + 'timestamp': mock.ANY, + 'message_id': mock.ANY}) + + def test_requeue(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + endpoint = mock.Mock() + endpoint.info = mock.Mock() + + def side_effect_requeue(*args, **kwargs): + if endpoint.info.call_count == 1: + return oslo_messaging.NotificationResult.REQUEUE + return oslo_messaging.NotificationResult.HANDLED + + endpoint.info.side_effect = side_effect_requeue + listener_thread = self._setup_listener(transport, + [endpoint], 2) + notifier = self._setup_notifier(transport) + notifier.info({}, 'an_event.start', 'test') + + self.assertFalse(listener_thread.wait_end()) + + endpoint.info.assert_has_calls([ + mock.call({}, 'testpublisher', 'an_event.start', 'test', + {'timestamp': mock.ANY, 'message_id': mock.ANY}), + mock.call({}, 'testpublisher', 'an_event.start', 'test', + {'timestamp': mock.ANY, 'message_id': mock.ANY})]) + + def test_two_pools(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + endpoint1 = mock.Mock() + endpoint1.info.return_value = None + endpoint2 = mock.Mock() + endpoint2.info.return_value = None + + targets = [oslo_messaging.Target(topic="topic")] + listener1_thread = self._setup_listener(transport, [endpoint1], 2, + targets=targets, pool="pool1") + listener2_thread = self._setup_listener(transport, [endpoint2], 2, + targets=targets, pool="pool2") + + notifier = self._setup_notifier(transport, topic="topic") + notifier.info({'ctxt': '0'}, 'an_event.start', 'test message0') + notifier.info({'ctxt': '1'}, 'an_event.start', 'test message1') + + self.assertFalse(listener2_thread.wait_end()) + self.assertFalse(listener1_thread.wait_end()) + + def mocked_endpoint_call(i): + return mock.call({'ctxt': '%d' % i}, 'testpublisher', + 'an_event.start', 'test message%d' % i, + {'timestamp': mock.ANY, 'message_id': mock.ANY}) + + endpoint1.info.assert_has_calls([mocked_endpoint_call(0), + mocked_endpoint_call(1)]) + endpoint2.info.assert_has_calls([mocked_endpoint_call(0), + mocked_endpoint_call(1)]) + + def test_two_pools_three_listener(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + endpoint1 = mock.Mock() + endpoint1.info.return_value = None + endpoint2 = mock.Mock() + endpoint2.info.return_value = None + endpoint3 = mock.Mock() + endpoint3.info.return_value = None + + targets = [oslo_messaging.Target(topic="topic")] + listener1_thread = self._setup_listener(transport, [endpoint1], 100, + targets=targets, pool="pool1") + listener2_thread = self._setup_listener(transport, [endpoint2], 100, + targets=targets, pool="pool2") + listener3_thread = self._setup_listener(transport, [endpoint3], 100, + targets=targets, pool="pool2") + + def mocked_endpoint_call(i): + return mock.call({'ctxt': '%d' % i}, 'testpublisher', + 'an_event.start', 'test message%d' % i, + {'timestamp': mock.ANY, 'message_id': mock.ANY}) + + notifier = self._setup_notifier(transport, topic="topic") + mocked_endpoint1_calls = [] + for i in range(0, 25): + notifier.info({'ctxt': '%d' % i}, 'an_event.start', + 'test message%d' % i) + mocked_endpoint1_calls.append(mocked_endpoint_call(i)) + + self.trackers['pool2'].wait_for(25) + listener2_thread.stop() + + for i in range(0, 25): + notifier.info({'ctxt': '%d' % i}, 'an_event.start', + 'test message%d' % i) + mocked_endpoint1_calls.append(mocked_endpoint_call(i)) + + self.trackers['pool2'].wait_for(50) + listener2_thread.start() + listener3_thread.stop() + + for i in range(0, 25): + notifier.info({'ctxt': '%d' % i}, 'an_event.start', + 'test message%d' % i) + mocked_endpoint1_calls.append(mocked_endpoint_call(i)) + + self.trackers['pool2'].wait_for(75) + listener3_thread.start() + + for i in range(0, 25): + notifier.info({'ctxt': '%d' % i}, 'an_event.start', + 'test message%d' % i) + mocked_endpoint1_calls.append(mocked_endpoint_call(i)) + + self.assertFalse(listener3_thread.wait_end()) + self.assertFalse(listener2_thread.wait_end()) + self.assertFalse(listener1_thread.wait_end()) + + self.assertEqual(100, endpoint1.info.call_count) + endpoint1.info.assert_has_calls(mocked_endpoint1_calls) + + self.assertLessEqual(25, endpoint2.info.call_count) + self.assertLessEqual(25, endpoint3.info.call_count) + + self.assertEqual(100, endpoint2.info.call_count + + endpoint3.info.call_count) + for call in mocked_endpoint1_calls: + self.assertIn(call, endpoint2.info.mock_calls + + endpoint3.info.mock_calls) diff --git a/oslo_messaging/tests/notify/test_log_handler.py b/oslo_messaging/tests/notify/test_log_handler.py new file mode 100644 index 000000000..671269735 --- /dev/null +++ b/oslo_messaging/tests/notify/test_log_handler.py @@ -0,0 +1,58 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + +import mock + +import oslo_messaging +from oslo_messaging.notify import log_handler +from oslo_messaging.tests.notify import test_notifier +from oslo_messaging.tests import utils as test_utils + + +class PublishErrorsHandlerTestCase(test_utils.BaseTestCase): + """Tests for log.PublishErrorsHandler""" + def setUp(self): + super(PublishErrorsHandlerTestCase, self).setUp() + self.publisherrorshandler = (log_handler. + PublishErrorsHandler(logging.ERROR)) + + def test_emit_cfg_log_notifier_in_notifier_drivers(self): + drivers = ['messaging', 'log'] + self.config(notification_driver=drivers) + self.stub_flg = True + + transport = test_notifier._FakeTransport(self.conf) + notifier = oslo_messaging.Notifier(transport) + + def fake_notifier(*args, **kwargs): + self.stub_flg = False + + self.stubs.Set(notifier, 'error', fake_notifier) + + logrecord = logging.LogRecord(name='name', level='WARN', + pathname='/tmp', lineno=1, msg='Message', + args=None, exc_info=None) + self.publisherrorshandler.emit(logrecord) + self.assertTrue(self.stub_flg) + + @mock.patch('oslo_messaging.notify.notifier.Notifier._notify') + def test_emit_notification(self, mock_notify): + logrecord = logging.LogRecord(name='name', level='ERROR', + pathname='/tmp', lineno=1, msg='Message', + args=None, exc_info=None) + self.publisherrorshandler.emit(logrecord) + self.assertEqual('error.publisher', + self.publisherrorshandler._notifier.publisher_id) + mock_notify.assert_called_with(None, 'error_notification', + {'error': 'Message'}, 'ERROR') diff --git a/oslo_messaging/tests/notify/test_logger.py b/oslo_messaging/tests/notify/test_logger.py new file mode 100644 index 000000000..c551493da --- /dev/null +++ b/oslo_messaging/tests/notify/test_logger.py @@ -0,0 +1,153 @@ +# Copyright 2013 eNovance +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime +import logging +import logging.config +import os +import sys + +import mock +import testscenarios +import testtools + +from oslo.utils import timeutils +import oslo_messaging +from oslo_messaging.tests.notify import test_notifier +from oslo_messaging.tests import utils as test_utils + + +load_tests = testscenarios.load_tests_apply_scenarios + +# Stolen from openstack.common.logging +logging.AUDIT = logging.INFO + 1 +logging.addLevelName(logging.AUDIT, 'AUDIT') + + +class TestLogNotifier(test_utils.BaseTestCase): + + scenarios = [ + ('debug', dict(priority='debug')), + ('info', dict(priority='info')), + ('warning', dict(priority='warning', queue='WARN')), + ('warn', dict(priority='warn')), + ('error', dict(priority='error')), + ('critical', dict(priority='critical')), + ('audit', dict(priority='audit')), + ] + + def setUp(self): + super(TestLogNotifier, self).setUp() + self.addCleanup(oslo_messaging.notify._impl_test.reset) + self.config(notification_driver=['test']) + # NOTE(jamespage) disable thread information logging for testing + # as this causes test failures when zmq tests monkey_patch via + # eventlet + logging.logThreads = 0 + + @mock.patch('oslo.utils.timeutils.utcnow') + def test_logger(self, mock_utcnow): + with mock.patch('oslo_messaging.transport.get_transport', + return_value=test_notifier._FakeTransport(self.conf)): + self.logger = oslo_messaging.LoggingNotificationHandler('test://') + + mock_utcnow.return_value = datetime.datetime.utcnow() + + levelno = getattr(logging, self.priority.upper(), 42) + + record = logging.LogRecord('foo', + levelno, + '/foo/bar', + 42, + 'Something happened', + None, + None) + + self.logger.emit(record) + + n = oslo_messaging.notify._impl_test.NOTIFICATIONS[0][1] + self.assertEqual(getattr(self, 'queue', self.priority.upper()), + n['priority']) + self.assertEqual('logrecord', n['event_type']) + self.assertEqual(str(timeutils.utcnow()), n['timestamp']) + self.assertEqual(None, n['publisher_id']) + self.assertEqual( + {'process': os.getpid(), + 'funcName': None, + 'name': 'foo', + 'thread': None, + 'levelno': levelno, + 'processName': 'MainProcess', + 'pathname': '/foo/bar', + 'lineno': 42, + 'msg': 'Something happened', + 'exc_info': None, + 'levelname': logging.getLevelName(levelno), + 'extra': None}, + n['payload']) + + @testtools.skipUnless(hasattr(logging.config, 'dictConfig'), + "Need logging.config.dictConfig (Python >= 2.7)") + @mock.patch('oslo.utils.timeutils.utcnow') + def test_logging_conf(self, mock_utcnow): + with mock.patch('oslo_messaging.transport.get_transport', + return_value=test_notifier._FakeTransport(self.conf)): + logging.config.dictConfig({ + 'version': 1, + 'handlers': { + 'notification': { + 'class': 'oslo_messaging.LoggingNotificationHandler', + 'level': self.priority.upper(), + 'url': 'test://', + }, + }, + 'loggers': { + 'default': { + 'handlers': ['notification'], + 'level': self.priority.upper(), + }, + }, + }) + + mock_utcnow.return_value = datetime.datetime.utcnow() + + levelno = getattr(logging, self.priority.upper()) + + logger = logging.getLogger('default') + lineno = sys._getframe().f_lineno + 1 + logger.log(levelno, 'foobar') + + n = oslo_messaging.notify._impl_test.NOTIFICATIONS[0][1] + self.assertEqual(getattr(self, 'queue', self.priority.upper()), + n['priority']) + self.assertEqual('logrecord', n['event_type']) + self.assertEqual(str(timeutils.utcnow()), n['timestamp']) + self.assertEqual(None, n['publisher_id']) + pathname = __file__ + if pathname.endswith(('.pyc', '.pyo')): + pathname = pathname[:-1] + self.assertDictEqual( + n['payload'], + {'process': os.getpid(), + 'funcName': 'test_logging_conf', + 'name': 'default', + 'thread': None, + 'levelno': levelno, + 'processName': 'MainProcess', + 'pathname': pathname, + 'lineno': lineno, + 'msg': 'foobar', + 'exc_info': None, + 'levelname': logging.getLevelName(levelno), + 'extra': None}) diff --git a/oslo_messaging/tests/notify/test_middleware.py b/oslo_messaging/tests/notify/test_middleware.py new file mode 100644 index 000000000..f98e6424c --- /dev/null +++ b/oslo_messaging/tests/notify/test_middleware.py @@ -0,0 +1,190 @@ +# Copyright 2013-2014 eNovance +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import uuid + +import mock +import webob + +from oslo_messaging.notify import middleware +from oslo_messaging.tests import utils + + +class FakeApp(object): + def __call__(self, env, start_response): + body = 'Some response' + start_response('200 OK', [ + ('Content-Type', 'text/plain'), + ('Content-Length', str(sum(map(len, body)))) + ]) + return [body] + + +class FakeFailingApp(object): + def __call__(self, env, start_response): + raise Exception("It happens!") + + +class NotifierMiddlewareTest(utils.BaseTestCase): + + def test_notification(self): + m = middleware.RequestNotifier(FakeApp()) + req = webob.Request.blank('/foo/bar', + environ={'REQUEST_METHOD': 'GET', + 'HTTP_X_AUTH_TOKEN': uuid.uuid4()}) + with mock.patch( + 'oslo_messaging.notify.notifier.Notifier._notify') as notify: + m(req) + # Check first notification with only 'request' + call_args = notify.call_args_list[0][0] + self.assertEqual(call_args[1], 'http.request') + self.assertEqual(call_args[3], 'INFO') + self.assertEqual(set(call_args[2].keys()), + set(['request'])) + + request = call_args[2]['request'] + self.assertEqual(request['PATH_INFO'], '/foo/bar') + self.assertEqual(request['REQUEST_METHOD'], 'GET') + self.assertIn('HTTP_X_SERVICE_NAME', request) + self.assertNotIn('HTTP_X_AUTH_TOKEN', request) + self.assertFalse(any(map(lambda s: s.startswith('wsgi.'), + request.keys())), + "WSGI fields are filtered out") + + # Check second notification with request + response + call_args = notify.call_args_list[1][0] + self.assertEqual(call_args[1], 'http.response') + self.assertEqual(call_args[3], 'INFO') + self.assertEqual(set(call_args[2].keys()), + set(['request', 'response'])) + + request = call_args[2]['request'] + self.assertEqual(request['PATH_INFO'], '/foo/bar') + self.assertEqual(request['REQUEST_METHOD'], 'GET') + self.assertIn('HTTP_X_SERVICE_NAME', request) + self.assertNotIn('HTTP_X_AUTH_TOKEN', request) + self.assertFalse(any(map(lambda s: s.startswith('wsgi.'), + request.keys())), + "WSGI fields are filtered out") + + response = call_args[2]['response'] + self.assertEqual(response['status'], '200 OK') + self.assertEqual(response['headers']['content-length'], '13') + + def test_notification_response_failure(self): + m = middleware.RequestNotifier(FakeFailingApp()) + req = webob.Request.blank('/foo/bar', + environ={'REQUEST_METHOD': 'GET', + 'HTTP_X_AUTH_TOKEN': uuid.uuid4()}) + with mock.patch( + 'oslo_messaging.notify.notifier.Notifier._notify') as notify: + try: + m(req) + self.fail("Application exception has not been re-raised") + except Exception: + pass + # Check first notification with only 'request' + call_args = notify.call_args_list[0][0] + self.assertEqual(call_args[1], 'http.request') + self.assertEqual(call_args[3], 'INFO') + self.assertEqual(set(call_args[2].keys()), + set(['request'])) + + request = call_args[2]['request'] + self.assertEqual(request['PATH_INFO'], '/foo/bar') + self.assertEqual(request['REQUEST_METHOD'], 'GET') + self.assertIn('HTTP_X_SERVICE_NAME', request) + self.assertNotIn('HTTP_X_AUTH_TOKEN', request) + self.assertFalse(any(map(lambda s: s.startswith('wsgi.'), + request.keys())), + "WSGI fields are filtered out") + + # Check second notification with 'request' and 'exception' + call_args = notify.call_args_list[1][0] + self.assertEqual(call_args[1], 'http.response') + self.assertEqual(call_args[3], 'INFO') + self.assertEqual(set(call_args[2].keys()), + set(['request', 'exception'])) + + request = call_args[2]['request'] + self.assertEqual(request['PATH_INFO'], '/foo/bar') + self.assertEqual(request['REQUEST_METHOD'], 'GET') + self.assertIn('HTTP_X_SERVICE_NAME', request) + self.assertNotIn('HTTP_X_AUTH_TOKEN', request) + self.assertFalse(any(map(lambda s: s.startswith('wsgi.'), + request.keys())), + "WSGI fields are filtered out") + + exception = call_args[2]['exception'] + self.assertIn('middleware.py', exception['traceback'][0]) + self.assertIn('It happens!', exception['traceback'][-1]) + self.assertEqual(exception['value'], "Exception('It happens!',)") + + def test_process_request_fail(self): + def notify_error(context, publisher_id, event_type, + priority, payload): + raise Exception('error') + with mock.patch('oslo_messaging.notify.notifier.Notifier._notify', + notify_error): + m = middleware.RequestNotifier(FakeApp()) + req = webob.Request.blank('/foo/bar', + environ={'REQUEST_METHOD': 'GET'}) + m.process_request(req) + + def test_process_response_fail(self): + def notify_error(context, publisher_id, event_type, + priority, payload): + raise Exception('error') + with mock.patch('oslo_messaging.notify.notifier.Notifier._notify', + notify_error): + m = middleware.RequestNotifier(FakeApp()) + req = webob.Request.blank('/foo/bar', + environ={'REQUEST_METHOD': 'GET'}) + m.process_response(req, webob.response.Response()) + + def test_ignore_req_opt(self): + m = middleware.RequestNotifier(FakeApp(), + ignore_req_list='get, PUT') + req = webob.Request.blank('/skip/foo', + environ={'REQUEST_METHOD': 'GET'}) + req1 = webob.Request.blank('/skip/foo', + environ={'REQUEST_METHOD': 'PUT'}) + req2 = webob.Request.blank('/accept/foo', + environ={'REQUEST_METHOD': 'POST'}) + with mock.patch( + 'oslo_messaging.notify.notifier.Notifier._notify') as notify: + # Check GET request does not send notification + m(req) + m(req1) + self.assertEqual(len(notify.call_args_list), 0) + + # Check non-GET request does send notification + m(req2) + self.assertEqual(len(notify.call_args_list), 2) + call_args = notify.call_args_list[0][0] + self.assertEqual(call_args[1], 'http.request') + self.assertEqual(call_args[3], 'INFO') + self.assertEqual(set(call_args[2].keys()), + set(['request'])) + + request = call_args[2]['request'] + self.assertEqual(request['PATH_INFO'], '/accept/foo') + self.assertEqual(request['REQUEST_METHOD'], 'POST') + + call_args = notify.call_args_list[1][0] + self.assertEqual(call_args[1], 'http.response') + self.assertEqual(call_args[3], 'INFO') + self.assertEqual(set(call_args[2].keys()), + set(['request', 'response'])) diff --git a/oslo_messaging/tests/notify/test_notifier.py b/oslo_messaging/tests/notify/test_notifier.py new file mode 100644 index 000000000..6b94d15e9 --- /dev/null +++ b/oslo_messaging/tests/notify/test_notifier.py @@ -0,0 +1,540 @@ + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime +import logging +import sys +import uuid + +import fixtures +import mock +from stevedore import dispatch +from stevedore import extension +import testscenarios +import yaml + +from oslo.serialization import jsonutils +from oslo.utils import timeutils +import oslo_messaging +from oslo_messaging.notify import _impl_log +from oslo_messaging.notify import _impl_messaging +from oslo_messaging.notify import _impl_test +from oslo_messaging.notify import notifier as msg_notifier +from oslo_messaging import serializer as msg_serializer +from oslo_messaging.tests import utils as test_utils + +load_tests = testscenarios.load_tests_apply_scenarios + + +class _FakeTransport(object): + + def __init__(self, conf): + self.conf = conf + + def _send_notification(self, target, ctxt, message, version, retry=None): + pass + + +class _ReRaiseLoggedExceptionsFixture(fixtures.Fixture): + + """Record logged exceptions and re-raise in cleanup. + + The notifier just logs notification send errors so, for the sake of + debugging test failures, we record any exceptions logged and re-raise them + during cleanup. + """ + + class FakeLogger(object): + + def __init__(self): + self.exceptions = [] + + def exception(self, msg, *args, **kwargs): + self.exceptions.append(sys.exc_info()[1]) + + def setUp(self): + super(_ReRaiseLoggedExceptionsFixture, self).setUp() + + self.logger = self.FakeLogger() + + def reraise_exceptions(): + for ex in self.logger.exceptions: + raise ex + + self.addCleanup(reraise_exceptions) + + +class TestMessagingNotifier(test_utils.BaseTestCase): + + _v1 = [ + ('v1', dict(v1=True)), + ('not_v1', dict(v1=False)), + ] + + _v2 = [ + ('v2', dict(v2=True)), + ('not_v2', dict(v2=False)), + ] + + _publisher_id = [ + ('ctor_pub_id', dict(ctor_pub_id='test', + expected_pub_id='test')), + ('prep_pub_id', dict(prep_pub_id='test.localhost', + expected_pub_id='test.localhost')), + ('override', dict(ctor_pub_id='test', + prep_pub_id='test.localhost', + expected_pub_id='test.localhost')), + ] + + _topics = [ + ('no_topics', dict(topics=[])), + ('single_topic', dict(topics=['notifications'])), + ('multiple_topic2', dict(topics=['foo', 'bar'])), + ] + + _priority = [ + ('audit', dict(priority='audit')), + ('debug', dict(priority='debug')), + ('info', dict(priority='info')), + ('warn', dict(priority='warn')), + ('error', dict(priority='error')), + ('sample', dict(priority='sample')), + ('critical', dict(priority='critical')), + ] + + _payload = [ + ('payload', dict(payload={'foo': 'bar'})), + ] + + _context = [ + ('ctxt', dict(ctxt={'user': 'bob'})), + ] + + _retry = [ + ('unconfigured', dict()), + ('None', dict(retry=None)), + ('0', dict(retry=0)), + ('5', dict(retry=5)), + ] + + @classmethod + def generate_scenarios(cls): + cls.scenarios = testscenarios.multiply_scenarios(cls._v1, + cls._v2, + cls._publisher_id, + cls._topics, + cls._priority, + cls._payload, + cls._context, + cls._retry) + + def setUp(self): + super(TestMessagingNotifier, self).setUp() + + self.logger = self.useFixture(_ReRaiseLoggedExceptionsFixture()).logger + self.stubs.Set(_impl_messaging, 'LOG', self.logger) + self.stubs.Set(msg_notifier, '_LOG', self.logger) + + @mock.patch('oslo.utils.timeutils.utcnow') + def test_notifier(self, mock_utcnow): + drivers = [] + if self.v1: + drivers.append('messaging') + if self.v2: + drivers.append('messagingv2') + + self.config(notification_driver=drivers, + notification_topics=self.topics) + + transport = _FakeTransport(self.conf) + + if hasattr(self, 'ctor_pub_id'): + notifier = oslo_messaging.Notifier(transport, + publisher_id=self.ctor_pub_id) + else: + notifier = oslo_messaging.Notifier(transport) + + prepare_kwds = {} + if hasattr(self, 'retry'): + prepare_kwds['retry'] = self.retry + if hasattr(self, 'prep_pub_id'): + prepare_kwds['publisher_id'] = self.prep_pub_id + if prepare_kwds: + notifier = notifier.prepare(**prepare_kwds) + + self.mox.StubOutWithMock(transport, '_send_notification') + + message_id = uuid.uuid4() + self.mox.StubOutWithMock(uuid, 'uuid4') + uuid.uuid4().AndReturn(message_id) + + mock_utcnow.return_value = datetime.datetime.utcnow() + + message = { + 'message_id': str(message_id), + 'publisher_id': self.expected_pub_id, + 'event_type': 'test.notify', + 'priority': self.priority.upper(), + 'payload': self.payload, + 'timestamp': str(timeutils.utcnow()), + } + + sends = [] + if self.v1: + sends.append(dict(version=1.0)) + if self.v2: + sends.append(dict(version=2.0)) + + for send_kwargs in sends: + for topic in self.topics: + if hasattr(self, 'retry'): + send_kwargs['retry'] = self.retry + else: + send_kwargs['retry'] = None + target = oslo_messaging.Target(topic='%s.%s' % (topic, + self.priority)) + transport._send_notification(target, self.ctxt, message, + **send_kwargs).InAnyOrder() + + self.mox.ReplayAll() + + method = getattr(notifier, self.priority) + method(self.ctxt, 'test.notify', self.payload) + + +TestMessagingNotifier.generate_scenarios() + + +class TestSerializer(test_utils.BaseTestCase): + + def setUp(self): + super(TestSerializer, self).setUp() + self.addCleanup(_impl_test.reset) + + @mock.patch('oslo.utils.timeutils.utcnow') + def test_serializer(self, mock_utcnow): + transport = _FakeTransport(self.conf) + + serializer = msg_serializer.NoOpSerializer() + + notifier = oslo_messaging.Notifier(transport, + 'test.localhost', + driver='test', + topic='test', + serializer=serializer) + + message_id = uuid.uuid4() + self.mox.StubOutWithMock(uuid, 'uuid4') + uuid.uuid4().AndReturn(message_id) + + mock_utcnow.return_value = datetime.datetime.utcnow() + + self.mox.StubOutWithMock(serializer, 'serialize_context') + self.mox.StubOutWithMock(serializer, 'serialize_entity') + serializer.serialize_context(dict(user='bob')).\ + AndReturn(dict(user='alice')) + serializer.serialize_entity(dict(user='bob'), 'bar').AndReturn('sbar') + + self.mox.ReplayAll() + + notifier.info(dict(user='bob'), 'test.notify', 'bar') + + message = { + 'message_id': str(message_id), + 'publisher_id': 'test.localhost', + 'event_type': 'test.notify', + 'priority': 'INFO', + 'payload': 'sbar', + 'timestamp': str(timeutils.utcnow()), + } + + self.assertEqual([(dict(user='alice'), message, 'INFO', None)], + _impl_test.NOTIFICATIONS) + + +class TestLogNotifier(test_utils.BaseTestCase): + + @mock.patch('oslo.utils.timeutils.utcnow') + def test_notifier(self, mock_utcnow): + self.config(notification_driver=['log']) + + transport = _FakeTransport(self.conf) + + notifier = oslo_messaging.Notifier(transport, 'test.localhost') + + message_id = uuid.uuid4() + self.mox.StubOutWithMock(uuid, 'uuid4') + uuid.uuid4().AndReturn(message_id) + + mock_utcnow.return_value = datetime.datetime.utcnow() + + message = { + 'message_id': str(message_id), + 'publisher_id': 'test.localhost', + 'event_type': 'test.notify', + 'priority': 'INFO', + 'payload': 'bar', + 'timestamp': str(timeutils.utcnow()), + } + + logger = self.mox.CreateMockAnything() + + self.mox.StubOutWithMock(logging, 'getLogger') + logging.getLogger('oslo.messaging.notification.test.notify').\ + AndReturn(logger) + + logger.info(jsonutils.dumps(message)) + + self.mox.ReplayAll() + + notifier.info({}, 'test.notify', 'bar') + + def test_sample_priority(self): + # Ensure logger drops sample-level notifications. + driver = _impl_log.LogDriver(None, None, None) + + logger = self.mox.CreateMock( + logging.getLogger('oslo.messaging.notification.foo')) + logger.sample = None + self.mox.StubOutWithMock(logging, 'getLogger') + logging.getLogger('oslo.messaging.notification.foo').\ + AndReturn(logger) + + self.mox.ReplayAll() + + msg = {'event_type': 'foo'} + driver.notify(None, msg, "sample", None) + + +class TestRoutingNotifier(test_utils.BaseTestCase): + def setUp(self): + super(TestRoutingNotifier, self).setUp() + self.config(notification_driver=['routing']) + + transport = _FakeTransport(self.conf) + self.notifier = oslo_messaging.Notifier(transport) + self.router = self.notifier._driver_mgr['routing'].obj + + def _fake_extension_manager(self, ext): + return extension.ExtensionManager.make_test_instance( + [extension.Extension('test', None, None, ext), ]) + + def _empty_extension_manager(self): + return extension.ExtensionManager.make_test_instance([]) + + def test_should_load_plugin(self): + self.router.used_drivers = set(["zoo", "blah"]) + ext = mock.MagicMock() + ext.name = "foo" + self.assertFalse(self.router._should_load_plugin(ext)) + ext.name = "zoo" + self.assertTrue(self.router._should_load_plugin(ext)) + + def test_load_notifiers_no_config(self): + # default routing_notifier_config="" + self.router._load_notifiers() + self.assertEqual({}, self.router.routing_groups) + self.assertEqual(0, len(self.router.used_drivers)) + + def test_load_notifiers_no_extensions(self): + self.config(routing_notifier_config="routing_notifier.yaml") + routing_config = r"" + config_file = mock.MagicMock() + config_file.return_value = routing_config + + with mock.patch.object(self.router, '_get_notifier_config_file', + config_file): + with mock.patch('stevedore.dispatch.DispatchExtensionManager', + return_value=self._empty_extension_manager()): + with mock.patch('oslo_messaging.notify.' + '_impl_routing.LOG') as mylog: + self.router._load_notifiers() + self.assertFalse(mylog.debug.called) + self.assertEqual({}, self.router.routing_groups) + + def test_load_notifiers_config(self): + self.config(routing_notifier_config="routing_notifier.yaml") + routing_config = r""" +group_1: + rpc : foo +group_2: + rpc : blah + """ + + config_file = mock.MagicMock() + config_file.return_value = routing_config + + with mock.patch.object(self.router, '_get_notifier_config_file', + config_file): + with mock.patch('stevedore.dispatch.DispatchExtensionManager', + return_value=self._fake_extension_manager( + mock.MagicMock())): + self.router._load_notifiers() + groups = list(self.router.routing_groups.keys()) + groups.sort() + self.assertEqual(['group_1', 'group_2'], groups) + + def test_get_drivers_for_message_accepted_events(self): + config = r""" +group_1: + rpc: + accepted_events: + - foo.* + - blah.zoo.* + - zip + """ + groups = yaml.load(config) + group = groups['group_1'] + + # No matching event ... + self.assertEqual([], + self.router._get_drivers_for_message( + group, "unknown", "info")) + + # Child of foo ... + self.assertEqual(['rpc'], + self.router._get_drivers_for_message( + group, "foo.1", "info")) + + # Foo itself ... + self.assertEqual([], + self.router._get_drivers_for_message( + group, "foo", "info")) + + # Child of blah.zoo + self.assertEqual(['rpc'], + self.router._get_drivers_for_message( + group, "blah.zoo.zing", "info")) + + def test_get_drivers_for_message_accepted_priorities(self): + config = r""" +group_1: + rpc: + accepted_priorities: + - info + - error + """ + groups = yaml.load(config) + group = groups['group_1'] + + # No matching priority + self.assertEqual([], + self.router._get_drivers_for_message( + group, None, "unknown")) + + # Info ... + self.assertEqual(['rpc'], + self.router._get_drivers_for_message( + group, None, "info")) + + # Error (to make sure the list is getting processed) ... + self.assertEqual(['rpc'], + self.router._get_drivers_for_message( + group, None, "error")) + + def test_get_drivers_for_message_both(self): + config = r""" +group_1: + rpc: + accepted_priorities: + - info + accepted_events: + - foo.* + driver_1: + accepted_priorities: + - info + driver_2: + accepted_events: + - foo.* + """ + groups = yaml.load(config) + group = groups['group_1'] + + # Valid event, but no matching priority + self.assertEqual(['driver_2'], + self.router._get_drivers_for_message( + group, 'foo.blah', "unknown")) + + # Valid priority, but no matching event + self.assertEqual(['driver_1'], + self.router._get_drivers_for_message( + group, 'unknown', "info")) + + # Happy day ... + x = self.router._get_drivers_for_message(group, 'foo.blah', "info") + x.sort() + self.assertEqual(['driver_1', 'driver_2', 'rpc'], x) + + def test_filter_func(self): + ext = mock.MagicMock() + ext.name = "rpc" + + # Good ... + self.assertTrue(self.router._filter_func(ext, {}, {}, 'info', + None, ['foo', 'rpc'])) + + # Bad + self.assertFalse(self.router._filter_func(ext, {}, {}, 'info', + None, ['foo'])) + + def test_notify(self): + self.router.routing_groups = {'group_1': None, 'group_2': None} + drivers_mock = mock.MagicMock() + drivers_mock.side_effect = [['rpc'], ['foo']] + + with mock.patch.object(self.router, 'plugin_manager') as pm: + with mock.patch.object(self.router, '_get_drivers_for_message', + drivers_mock): + self.notifier.info({}, 'my_event', {}) + self.assertEqual(sorted(['rpc', 'foo']), + sorted(pm.map.call_args[0][6])) + + def test_notify_filtered(self): + self.config(routing_notifier_config="routing_notifier.yaml") + routing_config = r""" +group_1: + rpc: + accepted_events: + - my_event + rpc2: + accepted_priorities: + - info + bar: + accepted_events: + - nothing + """ + config_file = mock.MagicMock() + config_file.return_value = routing_config + + rpc_driver = mock.Mock() + rpc2_driver = mock.Mock() + bar_driver = mock.Mock() + + pm = dispatch.DispatchExtensionManager.make_test_instance( + [extension.Extension('rpc', None, None, rpc_driver), + extension.Extension('rpc2', None, None, rpc2_driver), + extension.Extension('bar', None, None, bar_driver)], + ) + + with mock.patch.object(self.router, '_get_notifier_config_file', + config_file): + with mock.patch('stevedore.dispatch.DispatchExtensionManager', + return_value=pm): + self.notifier.info({}, 'my_event', {}) + self.assertFalse(bar_driver.info.called) + rpc_driver.notify.assert_called_once_with( + {}, mock.ANY, 'INFO', None) + rpc2_driver.notify.assert_called_once_with( + {}, mock.ANY, 'INFO', None) diff --git a/oslo_messaging/tests/rpc/__init__.py b/oslo_messaging/tests/rpc/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/oslo_messaging/tests/rpc/test_client.py b/oslo_messaging/tests/rpc/test_client.py new file mode 100644 index 000000000..2276e6b48 --- /dev/null +++ b/oslo_messaging/tests/rpc/test_client.py @@ -0,0 +1,519 @@ + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import testscenarios + +from oslo.config import cfg +import oslo_messaging +from oslo_messaging import exceptions +from oslo_messaging import serializer as msg_serializer +from oslo_messaging.tests import utils as test_utils + +load_tests = testscenarios.load_tests_apply_scenarios + + +class _FakeTransport(object): + + def __init__(self, conf): + self.conf = conf + + def _send(self, *args, **kwargs): + pass + + +class TestCastCall(test_utils.BaseTestCase): + + scenarios = [ + ('cast_no_ctxt_no_args', dict(call=False, ctxt={}, args={})), + ('call_no_ctxt_no_args', dict(call=True, ctxt={}, args={})), + ('cast_ctxt_and_args', + dict(call=False, + ctxt=dict(user='testuser', project='testtenant'), + args=dict(bar='blaa', foobar=11.01))), + ('call_ctxt_and_args', + dict(call=True, + ctxt=dict(user='testuser', project='testtenant'), + args=dict(bar='blaa', foobar=11.01))), + ] + + def test_cast_call(self): + self.config(rpc_response_timeout=None) + + transport = _FakeTransport(self.conf) + client = oslo_messaging.RPCClient(transport, oslo_messaging.Target()) + + self.mox.StubOutWithMock(transport, '_send') + + msg = dict(method='foo', args=self.args) + kwargs = {'retry': None} + if self.call: + kwargs['wait_for_reply'] = True + kwargs['timeout'] = None + + transport._send(oslo_messaging.Target(), self.ctxt, msg, **kwargs) + self.mox.ReplayAll() + + method = client.call if self.call else client.cast + method(self.ctxt, 'foo', **self.args) + + +class TestCastToTarget(test_utils.BaseTestCase): + + _base = [ + ('all_none', dict(ctor={}, prepare={}, expect={})), + ('ctor_exchange', + dict(ctor=dict(exchange='testexchange'), + prepare={}, + expect=dict(exchange='testexchange'))), + ('prepare_exchange', + dict(ctor={}, + prepare=dict(exchange='testexchange'), + expect=dict(exchange='testexchange'))), + ('prepare_exchange_none', + dict(ctor=dict(exchange='testexchange'), + prepare=dict(exchange=None), + expect={})), + ('both_exchange', + dict(ctor=dict(exchange='ctorexchange'), + prepare=dict(exchange='testexchange'), + expect=dict(exchange='testexchange'))), + ('ctor_topic', + dict(ctor=dict(topic='testtopic'), + prepare={}, + expect=dict(topic='testtopic'))), + ('prepare_topic', + dict(ctor={}, + prepare=dict(topic='testtopic'), + expect=dict(topic='testtopic'))), + ('prepare_topic_none', + dict(ctor=dict(topic='testtopic'), + prepare=dict(topic=None), + expect={})), + ('both_topic', + dict(ctor=dict(topic='ctortopic'), + prepare=dict(topic='testtopic'), + expect=dict(topic='testtopic'))), + ('ctor_namespace', + dict(ctor=dict(namespace='testnamespace'), + prepare={}, + expect=dict(namespace='testnamespace'))), + ('prepare_namespace', + dict(ctor={}, + prepare=dict(namespace='testnamespace'), + expect=dict(namespace='testnamespace'))), + ('prepare_namespace_none', + dict(ctor=dict(namespace='testnamespace'), + prepare=dict(namespace=None), + expect={})), + ('both_namespace', + dict(ctor=dict(namespace='ctornamespace'), + prepare=dict(namespace='testnamespace'), + expect=dict(namespace='testnamespace'))), + ('ctor_version', + dict(ctor=dict(version='testversion'), + prepare={}, + expect=dict(version='testversion'))), + ('prepare_version', + dict(ctor={}, + prepare=dict(version='testversion'), + expect=dict(version='testversion'))), + ('prepare_version_none', + dict(ctor=dict(version='testversion'), + prepare=dict(version=None), + expect={})), + ('both_version', + dict(ctor=dict(version='ctorversion'), + prepare=dict(version='testversion'), + expect=dict(version='testversion'))), + ('ctor_server', + dict(ctor=dict(server='testserver'), + prepare={}, + expect=dict(server='testserver'))), + ('prepare_server', + dict(ctor={}, + prepare=dict(server='testserver'), + expect=dict(server='testserver'))), + ('prepare_server_none', + dict(ctor=dict(server='testserver'), + prepare=dict(server=None), + expect={})), + ('both_server', + dict(ctor=dict(server='ctorserver'), + prepare=dict(server='testserver'), + expect=dict(server='testserver'))), + ('ctor_fanout', + dict(ctor=dict(fanout=True), + prepare={}, + expect=dict(fanout=True))), + ('prepare_fanout', + dict(ctor={}, + prepare=dict(fanout=True), + expect=dict(fanout=True))), + ('prepare_fanout_none', + dict(ctor=dict(fanout=True), + prepare=dict(fanout=None), + expect={})), + ('both_fanout', + dict(ctor=dict(fanout=True), + prepare=dict(fanout=False), + expect=dict(fanout=False))), + ] + + _prepare = [ + ('single_prepare', dict(double_prepare=False)), + ('double_prepare', dict(double_prepare=True)), + ] + + @classmethod + def generate_scenarios(cls): + cls.scenarios = testscenarios.multiply_scenarios(cls._base, + cls._prepare) + + def setUp(self): + super(TestCastToTarget, self).setUp(conf=cfg.ConfigOpts()) + + def test_cast_to_target(self): + target = oslo_messaging.Target(**self.ctor) + expect_target = oslo_messaging.Target(**self.expect) + + transport = _FakeTransport(self.conf) + client = oslo_messaging.RPCClient(transport, target) + + self.mox.StubOutWithMock(transport, '_send') + + msg = dict(method='foo', args={}) + if 'namespace' in self.expect: + msg['namespace'] = self.expect['namespace'] + if 'version' in self.expect: + msg['version'] = self.expect['version'] + transport._send(expect_target, {}, msg, retry=None) + + self.mox.ReplayAll() + + if self.prepare: + client = client.prepare(**self.prepare) + if self.double_prepare: + client = client.prepare(**self.prepare) + client.cast({}, 'foo') + + +TestCastToTarget.generate_scenarios() + + +_notset = object() + + +class TestCallTimeout(test_utils.BaseTestCase): + + scenarios = [ + ('all_none', + dict(confval=None, ctor=None, prepare=_notset, expect=None)), + ('confval', + dict(confval=21.1, ctor=None, prepare=_notset, expect=21.1)), + ('ctor', + dict(confval=None, ctor=21.1, prepare=_notset, expect=21.1)), + ('ctor_zero', + dict(confval=None, ctor=0, prepare=_notset, expect=0)), + ('prepare', + dict(confval=None, ctor=None, prepare=21.1, expect=21.1)), + ('prepare_override', + dict(confval=None, ctor=10.1, prepare=21.1, expect=21.1)), + ('prepare_zero', + dict(confval=None, ctor=None, prepare=0, expect=0)), + ] + + def test_call_timeout(self): + self.config(rpc_response_timeout=self.confval) + + transport = _FakeTransport(self.conf) + client = oslo_messaging.RPCClient(transport, oslo_messaging.Target(), + timeout=self.ctor) + + self.mox.StubOutWithMock(transport, '_send') + + msg = dict(method='foo', args={}) + kwargs = dict(wait_for_reply=True, timeout=self.expect, retry=None) + transport._send(oslo_messaging.Target(), {}, msg, **kwargs) + + self.mox.ReplayAll() + + if self.prepare is not _notset: + client = client.prepare(timeout=self.prepare) + client.call({}, 'foo') + + +class TestCallRetry(test_utils.BaseTestCase): + + scenarios = [ + ('all_none', dict(ctor=None, prepare=_notset, expect=None)), + ('ctor', dict(ctor=21, prepare=_notset, expect=21)), + ('ctor_zero', dict(ctor=0, prepare=_notset, expect=0)), + ('prepare', dict(ctor=None, prepare=21, expect=21)), + ('prepare_override', dict(ctor=10, prepare=21, expect=21)), + ('prepare_zero', dict(ctor=None, prepare=0, expect=0)), + ] + + def test_call_retry(self): + transport = _FakeTransport(self.conf) + client = oslo_messaging.RPCClient(transport, oslo_messaging.Target(), + retry=self.ctor) + + self.mox.StubOutWithMock(transport, '_send') + + msg = dict(method='foo', args={}) + kwargs = dict(wait_for_reply=True, timeout=60, + retry=self.expect) + transport._send(oslo_messaging.Target(), {}, msg, **kwargs) + + self.mox.ReplayAll() + + if self.prepare is not _notset: + client = client.prepare(retry=self.prepare) + client.call({}, 'foo') + + +class TestCallFanout(test_utils.BaseTestCase): + + scenarios = [ + ('target', dict(prepare=_notset, target={'fanout': True})), + ('prepare', dict(prepare={'fanout': True}, target={})), + ('both', dict(prepare={'fanout': True}, target={'fanout': True})), + ] + + def test_call_fanout(self): + transport = _FakeTransport(self.conf) + client = oslo_messaging.RPCClient(transport, + oslo_messaging.Target(**self.target)) + + if self.prepare is not _notset: + client = client.prepare(**self.prepare) + + self.assertRaises(exceptions.InvalidTarget, + client.call, {}, 'foo') + + +class TestSerializer(test_utils.BaseTestCase): + + scenarios = [ + ('cast', + dict(call=False, + ctxt=dict(user='bob'), + args=dict(a='a', b='b', c='c'), + retval=None)), + ('call', + dict(call=True, + ctxt=dict(user='bob'), + args=dict(a='a', b='b', c='c'), + retval='d')), + ] + + def test_call_serializer(self): + self.config(rpc_response_timeout=None) + + transport = _FakeTransport(self.conf) + serializer = msg_serializer.NoOpSerializer() + + client = oslo_messaging.RPCClient(transport, oslo_messaging.Target(), + serializer=serializer) + + self.mox.StubOutWithMock(transport, '_send') + + msg = dict(method='foo', + args=dict([(k, 's' + v) for k, v in self.args.items()])) + kwargs = dict(wait_for_reply=True, timeout=None) if self.call else {} + kwargs['retry'] = None + transport._send(oslo_messaging.Target(), + dict(user='alice'), + msg, + **kwargs).AndReturn(self.retval) + + self.mox.StubOutWithMock(serializer, 'serialize_entity') + self.mox.StubOutWithMock(serializer, 'deserialize_entity') + self.mox.StubOutWithMock(serializer, 'serialize_context') + + for arg in self.args: + serializer.serialize_entity(self.ctxt, arg).AndReturn('s' + arg) + + if self.call: + serializer.deserialize_entity(self.ctxt, self.retval).\ + AndReturn('d' + self.retval) + + serializer.serialize_context(self.ctxt).AndReturn(dict(user='alice')) + + self.mox.ReplayAll() + + method = client.call if self.call else client.cast + retval = method(self.ctxt, 'foo', **self.args) + if self.retval is not None: + self.assertEqual('d' + self.retval, retval) + + +class TestVersionCap(test_utils.BaseTestCase): + + _call_vs_cast = [ + ('call', dict(call=True)), + ('cast', dict(call=False)), + ] + + _cap_scenarios = [ + ('all_none', + dict(cap=None, prepare_cap=_notset, + version=None, prepare_version=_notset, + success=True)), + ('ctor_cap_ok', + dict(cap='1.1', prepare_cap=_notset, + version='1.0', prepare_version=_notset, + success=True)), + ('ctor_cap_override_ok', + dict(cap='2.0', prepare_cap='1.1', + version='1.0', prepare_version='1.0', + success=True)), + ('ctor_cap_override_none_ok', + dict(cap='1.1', prepare_cap=None, + version='1.0', prepare_version=_notset, + success=True)), + ('ctor_cap_minor_fail', + dict(cap='1.0', prepare_cap=_notset, + version='1.1', prepare_version=_notset, + success=False)), + ('ctor_cap_major_fail', + dict(cap='2.0', prepare_cap=_notset, + version=None, prepare_version='1.0', + success=False)), + ] + + @classmethod + def generate_scenarios(cls): + cls.scenarios = ( + testscenarios.multiply_scenarios(cls._call_vs_cast, + cls._cap_scenarios)) + + def test_version_cap(self): + self.config(rpc_response_timeout=None) + + transport = _FakeTransport(self.conf) + + target = oslo_messaging.Target(version=self.version) + client = oslo_messaging.RPCClient(transport, target, + version_cap=self.cap) + + if self.success: + self.mox.StubOutWithMock(transport, '_send') + + if self.prepare_version is not _notset: + target = target(version=self.prepare_version) + + msg = dict(method='foo', args={}) + if target.version is not None: + msg['version'] = target.version + + kwargs = {'retry': None} + if self.call: + kwargs['wait_for_reply'] = True + kwargs['timeout'] = None + + transport._send(target, {}, msg, **kwargs) + + self.mox.ReplayAll() + + prep_kwargs = {} + if self.prepare_cap is not _notset: + prep_kwargs['version_cap'] = self.prepare_cap + if self.prepare_version is not _notset: + prep_kwargs['version'] = self.prepare_version + if prep_kwargs: + client = client.prepare(**prep_kwargs) + + method = client.call if self.call else client.cast + try: + method({}, 'foo') + except Exception as ex: + self.assertIsInstance(ex, oslo_messaging.RPCVersionCapError, ex) + self.assertFalse(self.success) + else: + self.assertTrue(self.success) + + +TestVersionCap.generate_scenarios() + + +class TestCanSendVersion(test_utils.BaseTestCase): + + scenarios = [ + ('all_none', + dict(cap=None, prepare_cap=_notset, + version=None, prepare_version=_notset, + can_send_version=_notset, + can_send=True)), + ('ctor_cap_ok', + dict(cap='1.1', prepare_cap=_notset, + version='1.0', prepare_version=_notset, + can_send_version=_notset, + can_send=True)), + ('ctor_cap_override_ok', + dict(cap='2.0', prepare_cap='1.1', + version='1.0', prepare_version='1.0', + can_send_version=_notset, + can_send=True)), + ('ctor_cap_override_none_ok', + dict(cap='1.1', prepare_cap=None, + version='1.0', prepare_version=_notset, + can_send_version=_notset, + can_send=True)), + ('ctor_cap_can_send_ok', + dict(cap='1.1', prepare_cap=None, + version='1.0', prepare_version=_notset, + can_send_version='1.1', + can_send=True)), + ('ctor_cap_can_send_none_ok', + dict(cap='1.1', prepare_cap=None, + version='1.0', prepare_version=_notset, + can_send_version=None, + can_send=True)), + ('ctor_cap_minor_fail', + dict(cap='1.0', prepare_cap=_notset, + version='1.1', prepare_version=_notset, + can_send_version=_notset, + can_send=False)), + ('ctor_cap_major_fail', + dict(cap='2.0', prepare_cap=_notset, + version=None, prepare_version='1.0', + can_send_version=_notset, + can_send=False)), + ] + + def test_version_cap(self): + self.config(rpc_response_timeout=None) + + transport = _FakeTransport(self.conf) + + target = oslo_messaging.Target(version=self.version) + client = oslo_messaging.RPCClient(transport, target, + version_cap=self.cap) + + prep_kwargs = {} + if self.prepare_cap is not _notset: + prep_kwargs['version_cap'] = self.prepare_cap + if self.prepare_version is not _notset: + prep_kwargs['version'] = self.prepare_version + if prep_kwargs: + client = client.prepare(**prep_kwargs) + + if self.can_send_version is not _notset: + can_send = client.can_send_version(version=self.can_send_version) + else: + can_send = client.can_send_version() + + self.assertEqual(self.can_send, can_send) diff --git a/oslo_messaging/tests/rpc/test_dispatcher.py b/oslo_messaging/tests/rpc/test_dispatcher.py new file mode 100644 index 000000000..edc4e7e0a --- /dev/null +++ b/oslo_messaging/tests/rpc/test_dispatcher.py @@ -0,0 +1,180 @@ + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import testscenarios + +import oslo_messaging +from oslo_messaging import serializer as msg_serializer +from oslo_messaging.tests import utils as test_utils + +load_tests = testscenarios.load_tests_apply_scenarios + + +class _FakeEndpoint(object): + + def __init__(self, target=None): + self.target = target + + def foo(self, ctxt, **kwargs): + pass + + def bar(self, ctxt, **kwargs): + pass + + +class TestDispatcher(test_utils.BaseTestCase): + + scenarios = [ + ('no_endpoints', + dict(endpoints=[], + dispatch_to=None, + ctxt={}, msg=dict(method='foo'), + success=False, ex=oslo_messaging.UnsupportedVersion)), + ('default_target', + dict(endpoints=[{}], + dispatch_to=dict(endpoint=0, method='foo'), + ctxt={}, msg=dict(method='foo'), + success=True, ex=None)), + ('default_target_ctxt_and_args', + dict(endpoints=[{}], + dispatch_to=dict(endpoint=0, method='bar'), + ctxt=dict(user='bob'), msg=dict(method='bar', + args=dict(blaa=True)), + success=True, ex=None)), + ('default_target_namespace', + dict(endpoints=[{}], + dispatch_to=dict(endpoint=0, method='foo'), + ctxt={}, msg=dict(method='foo', namespace=None), + success=True, ex=None)), + ('default_target_version', + dict(endpoints=[{}], + dispatch_to=dict(endpoint=0, method='foo'), + ctxt={}, msg=dict(method='foo', version='1.0'), + success=True, ex=None)), + ('default_target_no_such_method', + dict(endpoints=[{}], + dispatch_to=None, + ctxt={}, msg=dict(method='foobar'), + success=False, ex=oslo_messaging.NoSuchMethod)), + ('namespace', + dict(endpoints=[{}, dict(namespace='testns')], + dispatch_to=dict(endpoint=1, method='foo'), + ctxt={}, msg=dict(method='foo', namespace='testns'), + success=True, ex=None)), + ('namespace_mismatch', + dict(endpoints=[{}, dict(namespace='testns')], + dispatch_to=None, + ctxt={}, msg=dict(method='foo', namespace='nstest'), + success=False, ex=oslo_messaging.UnsupportedVersion)), + ('version', + dict(endpoints=[dict(version='1.5'), dict(version='3.4')], + dispatch_to=dict(endpoint=1, method='foo'), + ctxt={}, msg=dict(method='foo', version='3.2'), + success=True, ex=None)), + ('version_mismatch', + dict(endpoints=[dict(version='1.5'), dict(version='3.0')], + dispatch_to=None, + ctxt={}, msg=dict(method='foo', version='3.2'), + success=False, ex=oslo_messaging.UnsupportedVersion)), + ] + + def test_dispatcher(self): + endpoints = [mock.Mock(spec=_FakeEndpoint, + target=oslo_messaging.Target(**e)) + for e in self.endpoints] + + serializer = None + target = oslo_messaging.Target() + dispatcher = oslo_messaging.RPCDispatcher(target, endpoints, + serializer) + + def check_reply(reply=None, failure=None, log_failure=True): + if self.ex and failure is not None: + ex = failure[1] + self.assertFalse(self.success, ex) + self.assertIsNotNone(self.ex, ex) + self.assertIsInstance(ex, self.ex, ex) + if isinstance(ex, oslo_messaging.NoSuchMethod): + self.assertEqual(self.msg.get('method'), ex.method) + elif isinstance(ex, oslo_messaging.UnsupportedVersion): + self.assertEqual(self.msg.get('version', '1.0'), + ex.version) + if ex.method: + self.assertEqual(self.msg.get('method'), ex.method) + else: + self.assertTrue(self.success, failure) + self.assertIsNone(failure) + + incoming = mock.Mock(ctxt=self.ctxt, message=self.msg) + incoming.reply.side_effect = check_reply + + with dispatcher(incoming) as callback: + callback() + + for n, endpoint in enumerate(endpoints): + for method_name in ['foo', 'bar']: + method = getattr(endpoint, method_name) + if self.dispatch_to and n == self.dispatch_to['endpoint'] and \ + method_name == self.dispatch_to['method']: + method.assert_called_once_with( + self.ctxt, **self.msg.get('args', {})) + else: + self.assertEqual(0, method.call_count) + + self.assertEqual(1, incoming.reply.call_count) + + +class TestSerializer(test_utils.BaseTestCase): + + scenarios = [ + ('no_args_or_retval', + dict(ctxt={}, dctxt={}, args={}, retval=None)), + ('args_and_retval', + dict(ctxt=dict(user='bob'), + dctxt=dict(user='alice'), + args=dict(a='a', b='b', c='c'), + retval='d')), + ] + + def test_serializer(self): + endpoint = _FakeEndpoint() + serializer = msg_serializer.NoOpSerializer() + target = oslo_messaging.Target() + dispatcher = oslo_messaging.RPCDispatcher(target, [endpoint], + serializer) + + self.mox.StubOutWithMock(endpoint, 'foo') + args = dict([(k, 'd' + v) for k, v in self.args.items()]) + endpoint.foo(self.dctxt, **args).AndReturn(self.retval) + + self.mox.StubOutWithMock(serializer, 'serialize_entity') + self.mox.StubOutWithMock(serializer, 'deserialize_entity') + self.mox.StubOutWithMock(serializer, 'deserialize_context') + + serializer.deserialize_context(self.ctxt).AndReturn(self.dctxt) + + for arg in self.args: + serializer.deserialize_entity(self.dctxt, arg).AndReturn('d' + arg) + + serializer.serialize_entity(self.dctxt, self.retval).\ + AndReturn('s' + self.retval if self.retval else None) + + self.mox.ReplayAll() + + retval = dispatcher._dispatch(self.ctxt, dict(method='foo', + args=self.args)) + if self.retval is not None: + self.assertEqual('s' + self.retval, retval) diff --git a/oslo_messaging/tests/rpc/test_server.py b/oslo_messaging/tests/rpc/test_server.py new file mode 100644 index 000000000..3970b95a4 --- /dev/null +++ b/oslo_messaging/tests/rpc/test_server.py @@ -0,0 +1,504 @@ + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import threading + +import mock +import testscenarios + +from oslo.config import cfg +import oslo_messaging +from oslo_messaging.tests import utils as test_utils + +load_tests = testscenarios.load_tests_apply_scenarios + + +class ServerSetupMixin(object): + + class Server(object): + def __init__(self, transport, topic, server, endpoint, serializer): + target = oslo_messaging.Target(topic=topic, server=server) + self._server = oslo_messaging.get_rpc_server(transport, + target, + [endpoint, self], + serializer=serializer) + + def stop(self, ctxt): + # Check start() does nothing with a running server + self._server.start() + self._server.stop() + self._server.wait() + + def start(self): + self._server.start() + + class TestSerializer(object): + + def serialize_entity(self, ctxt, entity): + return ('s' + entity) if entity else entity + + def deserialize_entity(self, ctxt, entity): + return ('d' + entity) if entity else entity + + def serialize_context(self, ctxt): + return dict([(k, 's' + v) for k, v in ctxt.items()]) + + def deserialize_context(self, ctxt): + return dict([(k, 'd' + v) for k, v in ctxt.items()]) + + def __init__(self): + self.serializer = self.TestSerializer() + + def _setup_server(self, transport, endpoint, topic=None, server=None): + server = self.Server(transport, + topic=topic or 'testtopic', + server=server or 'testserver', + endpoint=endpoint, + serializer=self.serializer) + + thread = threading.Thread(target=server.start) + thread.daemon = True + thread.start() + + return thread + + def _stop_server(self, client, server_thread, topic=None): + if topic is not None: + client = client.prepare(topic=topic) + client.cast({}, 'stop') + server_thread.join(timeout=30) + + def _setup_client(self, transport, topic='testtopic'): + return oslo_messaging.RPCClient(transport, + oslo_messaging.Target(topic=topic), + serializer=self.serializer) + + +class TestRPCServer(test_utils.BaseTestCase, ServerSetupMixin): + + def __init__(self, *args): + super(TestRPCServer, self).__init__(*args) + ServerSetupMixin.__init__(self) + + def setUp(self): + super(TestRPCServer, self).setUp(conf=cfg.ConfigOpts()) + + def test_constructor(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + target = oslo_messaging.Target(topic='foo', server='bar') + endpoints = [object()] + serializer = object() + + server = oslo_messaging.get_rpc_server(transport, target, endpoints, + serializer=serializer) + + self.assertIs(server.conf, self.conf) + self.assertIs(server.transport, transport) + self.assertIsInstance(server.dispatcher, oslo_messaging.RPCDispatcher) + self.assertIs(server.dispatcher.endpoints, endpoints) + self.assertIs(server.dispatcher.serializer, serializer) + self.assertEqual('blocking', server.executor) + + def test_server_wait_method(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + target = oslo_messaging.Target(topic='foo', server='bar') + endpoints = [object()] + serializer = object() + + server = oslo_messaging.get_rpc_server(transport, target, endpoints, + serializer=serializer) + # Mocking executor + server._executor = mock.Mock() + # Here assigning executor's listener object to listener variable + # before calling wait method, beacuse in wait method we are + # setting executor to None. + listener = server._executor.listener + # call server wait method + server.wait() + self.assertIsNone(server._executor) + self.assertEqual(1, listener.cleanup.call_count) + + def test_no_target_server(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + server = oslo_messaging.get_rpc_server( + transport, + oslo_messaging.Target(topic='testtopic'), + []) + try: + server.start() + except Exception as ex: + self.assertIsInstance(ex, oslo_messaging.InvalidTarget, ex) + self.assertEqual('testtopic', ex.target.topic) + else: + self.assertTrue(False) + + def test_no_server_topic(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + target = oslo_messaging.Target(server='testserver') + server = oslo_messaging.get_rpc_server(transport, target, []) + try: + server.start() + except Exception as ex: + self.assertIsInstance(ex, oslo_messaging.InvalidTarget, ex) + self.assertEqual('testserver', ex.target.server) + else: + self.assertTrue(False) + + def _test_no_client_topic(self, call=True): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + client = self._setup_client(transport, topic=None) + + method = client.call if call else client.cast + + try: + method({}, 'ping', arg='foo') + except Exception as ex: + self.assertIsInstance(ex, oslo_messaging.InvalidTarget, ex) + self.assertIsNotNone(ex.target) + else: + self.assertTrue(False) + + def test_no_client_topic_call(self): + self._test_no_client_topic(call=True) + + def test_no_client_topic_cast(self): + self._test_no_client_topic(call=False) + + def test_client_call_timeout(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + finished = False + wait = threading.Condition() + + class TestEndpoint(object): + def ping(self, ctxt, arg): + with wait: + if not finished: + wait.wait() + + server_thread = self._setup_server(transport, TestEndpoint()) + client = self._setup_client(transport) + + try: + client.prepare(timeout=0).call({}, 'ping', arg='foo') + except Exception as ex: + self.assertIsInstance(ex, oslo_messaging.MessagingTimeout, ex) + else: + self.assertTrue(False) + + with wait: + finished = True + wait.notify() + + self._stop_server(client, server_thread) + + def test_unknown_executor(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + try: + oslo_messaging.get_rpc_server(transport, None, [], executor='foo') + except Exception as ex: + self.assertIsInstance(ex, oslo_messaging.ExecutorLoadFailure) + self.assertEqual('foo', ex.executor) + else: + self.assertTrue(False) + + def test_cast(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + class TestEndpoint(object): + def __init__(self): + self.pings = [] + + def ping(self, ctxt, arg): + self.pings.append(arg) + + endpoint = TestEndpoint() + server_thread = self._setup_server(transport, endpoint) + client = self._setup_client(transport) + + client.cast({}, 'ping', arg='foo') + client.cast({}, 'ping', arg='bar') + + self._stop_server(client, server_thread) + + self.assertEqual(['dsfoo', 'dsbar'], endpoint.pings) + + def test_call(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + class TestEndpoint(object): + def ping(self, ctxt, arg): + return arg + + server_thread = self._setup_server(transport, TestEndpoint()) + client = self._setup_client(transport) + + self.assertIsNone(client.call({}, 'ping', arg=None)) + self.assertEqual(0, client.call({}, 'ping', arg=0)) + self.assertEqual(False, client.call({}, 'ping', arg=False)) + self.assertEqual([], client.call({}, 'ping', arg=[])) + self.assertEqual({}, client.call({}, 'ping', arg={})) + self.assertEqual('dsdsfoo', client.call({}, 'ping', arg='foo')) + + self._stop_server(client, server_thread) + + def test_direct_call(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + class TestEndpoint(object): + def ping(self, ctxt, arg): + return arg + + server_thread = self._setup_server(transport, TestEndpoint()) + client = self._setup_client(transport) + + direct = client.prepare(server='testserver') + self.assertIsNone(direct.call({}, 'ping', arg=None)) + self.assertEqual(0, client.call({}, 'ping', arg=0)) + self.assertEqual(False, client.call({}, 'ping', arg=False)) + self.assertEqual([], client.call({}, 'ping', arg=[])) + self.assertEqual({}, client.call({}, 'ping', arg={})) + self.assertEqual('dsdsfoo', direct.call({}, 'ping', arg='foo')) + + self._stop_server(client, server_thread) + + def test_context(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + class TestEndpoint(object): + def ctxt_check(self, ctxt, key): + return ctxt[key] + + server_thread = self._setup_server(transport, TestEndpoint()) + client = self._setup_client(transport) + + self.assertEqual('dsdsb', + client.call({'dsa': 'b'}, + 'ctxt_check', + key='a')) + + self._stop_server(client, server_thread) + + def test_failure(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + class TestEndpoint(object): + def ping(self, ctxt, arg): + raise ValueError(arg) + + server_thread = self._setup_server(transport, TestEndpoint()) + client = self._setup_client(transport) + + try: + client.call({}, 'ping', arg='foo') + except Exception as ex: + self.assertIsInstance(ex, ValueError) + self.assertEqual('dsfoo', str(ex)) + else: + self.assertTrue(False) + + self._stop_server(client, server_thread) + + def test_expected_failure(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + class TestEndpoint(object): + @oslo_messaging.expected_exceptions(ValueError) + def ping(self, ctxt, arg): + raise ValueError(arg) + + server_thread = self._setup_server(transport, TestEndpoint()) + client = self._setup_client(transport) + + try: + client.call({}, 'ping', arg='foo') + except Exception as ex: + self.assertIsInstance(ex, ValueError) + self.assertEqual('dsfoo', str(ex)) + else: + self.assertTrue(False) + + self._stop_server(client, server_thread) + + +class TestMultipleServers(test_utils.BaseTestCase, ServerSetupMixin): + + _exchanges = [ + ('same_exchange', dict(exchange1=None, exchange2=None)), + ('diff_exchange', dict(exchange1='x1', exchange2='x2')), + ] + + _topics = [ + ('same_topic', dict(topic1='t', topic2='t')), + ('diff_topic', dict(topic1='t1', topic2='t2')), + ] + + _server = [ + ('same_server', dict(server1=None, server2=None)), + ('diff_server', dict(server1='s1', server2='s2')), + ] + + _fanout = [ + ('not_fanout', dict(fanout1=None, fanout2=None)), + ('fanout', dict(fanout1=True, fanout2=True)), + ] + + _method = [ + ('call', dict(call1=True, call2=True)), + ('cast', dict(call1=False, call2=False)), + ] + + _endpoints = [ + ('one_endpoint', + dict(multi_endpoints=False, + expect1=['ds1', 'ds2'], + expect2=['ds1', 'ds2'])), + ('two_endpoints', + dict(multi_endpoints=True, + expect1=['ds1'], + expect2=['ds2'])), + ] + + @classmethod + def generate_scenarios(cls): + cls.scenarios = testscenarios.multiply_scenarios(cls._exchanges, + cls._topics, + cls._server, + cls._fanout, + cls._method, + cls._endpoints) + + # fanout call not supported + def filter_fanout_call(scenario): + params = scenario[1] + fanout = params['fanout1'] or params['fanout2'] + call = params['call1'] or params['call2'] + return not (call and fanout) + + # listening multiple times on same topic/server pair not supported + def filter_same_topic_and_server(scenario): + params = scenario[1] + single_topic = params['topic1'] == params['topic2'] + single_server = params['server1'] == params['server2'] + return not (single_topic and single_server) + + # fanout to multiple servers on same topic and exchange + # each endpoint will receive both messages + def fanout_to_servers(scenario): + params = scenario[1] + fanout = params['fanout1'] or params['fanout2'] + single_exchange = params['exchange1'] == params['exchange2'] + single_topic = params['topic1'] == params['topic2'] + multi_servers = params['server1'] != params['server2'] + if fanout and single_exchange and single_topic and multi_servers: + params['expect1'] = params['expect1'][:] + params['expect1'] + params['expect2'] = params['expect2'][:] + params['expect2'] + return scenario + + # multiple endpoints on same topic and exchange + # either endpoint can get either message + def single_topic_multi_endpoints(scenario): + params = scenario[1] + single_exchange = params['exchange1'] == params['exchange2'] + single_topic = params['topic1'] == params['topic2'] + if single_topic and single_exchange and params['multi_endpoints']: + params['expect_either'] = (params['expect1'] + + params['expect2']) + params['expect1'] = params['expect2'] = [] + else: + params['expect_either'] = [] + return scenario + + for f in [filter_fanout_call, filter_same_topic_and_server]: + cls.scenarios = filter(f, cls.scenarios) + for m in [fanout_to_servers, single_topic_multi_endpoints]: + cls.scenarios = map(m, cls.scenarios) + + def __init__(self, *args): + super(TestMultipleServers, self).__init__(*args) + ServerSetupMixin.__init__(self) + + def setUp(self): + super(TestMultipleServers, self).setUp(conf=cfg.ConfigOpts()) + + def test_multiple_servers(self): + url1 = 'fake:///' + (self.exchange1 or '') + url2 = 'fake:///' + (self.exchange2 or '') + + transport1 = oslo_messaging.get_transport(self.conf, url=url1) + if url1 != url2: + transport2 = oslo_messaging.get_transport(self.conf, url=url1) + else: + transport2 = transport1 + + class TestEndpoint(object): + def __init__(self): + self.pings = [] + + def ping(self, ctxt, arg): + self.pings.append(arg) + + def alive(self, ctxt): + return 'alive' + + if self.multi_endpoints: + endpoint1, endpoint2 = TestEndpoint(), TestEndpoint() + else: + endpoint1 = endpoint2 = TestEndpoint() + + thread1 = self._setup_server(transport1, endpoint1, + topic=self.topic1, server=self.server1) + thread2 = self._setup_server(transport2, endpoint2, + topic=self.topic2, server=self.server2) + + client1 = self._setup_client(transport1, topic=self.topic1) + client2 = self._setup_client(transport2, topic=self.topic2) + + client1 = client1.prepare(server=self.server1) + client2 = client2.prepare(server=self.server2) + + if self.fanout1: + client1.call({}, 'alive') + client1 = client1.prepare(fanout=True) + if self.fanout2: + client2.call({}, 'alive') + client2 = client2.prepare(fanout=True) + + (client1.call if self.call1 else client1.cast)({}, 'ping', arg='1') + (client2.call if self.call2 else client2.cast)({}, 'ping', arg='2') + + self.assertTrue(thread1.isAlive()) + self._stop_server(client1.prepare(fanout=None), + thread1, topic=self.topic1) + self.assertTrue(thread2.isAlive()) + self._stop_server(client2.prepare(fanout=None), + thread2, topic=self.topic2) + + def check(pings, expect): + self.assertEqual(len(expect), len(pings)) + for a in expect: + self.assertIn(a, pings) + + if self.expect_either: + check(endpoint1.pings + endpoint2.pings, self.expect_either) + else: + check(endpoint1.pings, self.expect1) + check(endpoint2.pings, self.expect2) + + +TestMultipleServers.generate_scenarios() diff --git a/oslo_messaging/tests/test_amqp_driver.py b/oslo_messaging/tests/test_amqp_driver.py new file mode 100644 index 000000000..213a83216 --- /dev/null +++ b/oslo_messaging/tests/test_amqp_driver.py @@ -0,0 +1,729 @@ +# Copyright (C) 2014 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import select +import socket +import threading +import time +import uuid + +from six import moves +import testtools + +from oslo.utils import importutils +import oslo_messaging +from oslo_messaging.tests import utils as test_utils + +# TODO(kgiusti) Conditionally run these tests only if the necessary +# dependencies are installed. This should be removed once the proton libraries +# are available in the base repos for all supported platforms. +pyngus = importutils.try_import("pyngus") +if pyngus: + from oslo_messaging._drivers.protocols.amqp import driver as amqp_driver + + +LOG = logging.getLogger(__name__) + + +class _ListenerThread(threading.Thread): + """Run a blocking listener in a thread.""" + def __init__(self, listener, msg_count): + super(_ListenerThread, self).__init__() + self.listener = listener + self.msg_count = msg_count + self.messages = moves.queue.Queue() + self.daemon = True + self.start() + + def run(self): + LOG.info("Listener started") + while self.msg_count > 0: + in_msg = self.listener.poll() + self.messages.put(in_msg) + self.msg_count -= 1 + if in_msg.message.get('method') == 'echo': + in_msg.reply(reply={'correlation-id': + in_msg.message.get('id')}) + LOG.info("Listener stopped") + + def get_messages(self): + """Returns a list of all received messages.""" + msgs = [] + try: + while True: + m = self.messages.get(False) + msgs.append(m) + except moves.queue.Empty: + pass + return msgs + + +@testtools.skipUnless(pyngus, "proton modules not present") +class TestProtonDriverLoad(test_utils.BaseTestCase): + + def setUp(self): + super(TestProtonDriverLoad, self).setUp() + self.messaging_conf.transport_driver = 'amqp' + + def test_driver_load(self): + transport = oslo_messaging.get_transport(self.conf) + self.assertIsInstance(transport._driver, + amqp_driver.ProtonDriver) + + +class _AmqpBrokerTestCase(test_utils.BaseTestCase): + + @testtools.skipUnless(pyngus, "proton modules not present") + def setUp(self): + LOG.info("Starting Broker Test") + super(_AmqpBrokerTestCase, self).setUp() + self._broker = FakeBroker() + self._broker_addr = "amqp://%s:%d" % (self._broker.host, + self._broker.port) + self._broker_url = oslo_messaging.TransportURL.parse( + self.conf, self._broker_addr) + self._broker.start() + + def tearDown(self): + super(_AmqpBrokerTestCase, self).tearDown() + self._broker.stop() + LOG.info("Broker Test Ended") + + +class TestAmqpSend(_AmqpBrokerTestCase): + """Test sending and receiving messages.""" + + def test_driver_unconnected_cleanup(self): + """Verify the driver can cleanly shutdown even if never connected.""" + driver = amqp_driver.ProtonDriver(self.conf, self._broker_url) + driver.cleanup() + + def test_listener_cleanup(self): + """Verify unused listener can cleanly shutdown.""" + driver = amqp_driver.ProtonDriver(self.conf, self._broker_url) + target = oslo_messaging.Target(topic="test-topic") + listener = driver.listen(target) + self.assertIsInstance(listener, amqp_driver.ProtonListener) + driver.cleanup() + + def test_send_no_reply(self): + driver = amqp_driver.ProtonDriver(self.conf, self._broker_url) + target = oslo_messaging.Target(topic="test-topic") + listener = _ListenerThread(driver.listen(target), 1) + rc = driver.send(target, {"context": True}, + {"msg": "value"}, wait_for_reply=False) + self.assertIsNone(rc) + listener.join(timeout=30) + self.assertFalse(listener.isAlive()) + self.assertEqual(listener.messages.get().message, {"msg": "value"}) + driver.cleanup() + + def test_send_exchange_with_reply(self): + driver = amqp_driver.ProtonDriver(self.conf, self._broker_url) + target1 = oslo_messaging.Target(topic="test-topic", exchange="e1") + listener1 = _ListenerThread(driver.listen(target1), 1) + target2 = oslo_messaging.Target(topic="test-topic", exchange="e2") + listener2 = _ListenerThread(driver.listen(target2), 1) + + rc = driver.send(target1, {"context": "whatever"}, + {"method": "echo", "id": "e1"}, + wait_for_reply=True, + timeout=30) + self.assertIsNotNone(rc) + self.assertEqual(rc.get('correlation-id'), 'e1') + + rc = driver.send(target2, {"context": "whatever"}, + {"method": "echo", "id": "e2"}, + wait_for_reply=True, + timeout=30) + self.assertIsNotNone(rc) + self.assertEqual(rc.get('correlation-id'), 'e2') + + listener1.join(timeout=30) + self.assertFalse(listener1.isAlive()) + listener2.join(timeout=30) + self.assertFalse(listener2.isAlive()) + driver.cleanup() + + def test_messaging_patterns(self): + """Verify the direct, shared, and fanout message patterns work.""" + driver = amqp_driver.ProtonDriver(self.conf, self._broker_url) + target1 = oslo_messaging.Target(topic="test-topic", server="server1") + listener1 = _ListenerThread(driver.listen(target1), 4) + target2 = oslo_messaging.Target(topic="test-topic", server="server2") + listener2 = _ListenerThread(driver.listen(target2), 3) + + shared_target = oslo_messaging.Target(topic="test-topic") + fanout_target = oslo_messaging.Target(topic="test-topic", + fanout=True) + # this should go to only one server: + driver.send(shared_target, {"context": "whatever"}, + {"method": "echo", "id": "either-1"}, + wait_for_reply=True) + self.assertEqual(self._broker.topic_count, 1) + self.assertEqual(self._broker.direct_count, 1) # reply + + # this should go to the other server: + driver.send(shared_target, {"context": "whatever"}, + {"method": "echo", "id": "either-2"}, + wait_for_reply=True) + self.assertEqual(self._broker.topic_count, 2) + self.assertEqual(self._broker.direct_count, 2) # reply + + # these should only go to listener1: + driver.send(target1, {"context": "whatever"}, + {"method": "echo", "id": "server1-1"}, + wait_for_reply=True) + + driver.send(target1, {"context": "whatever"}, + {"method": "echo", "id": "server1-2"}, + wait_for_reply=True) + self.assertEqual(self._broker.direct_count, 6) # 2X(send+reply) + + # this should only go to listener2: + driver.send(target2, {"context": "whatever"}, + {"method": "echo", "id": "server2"}, + wait_for_reply=True) + self.assertEqual(self._broker.direct_count, 8) + + # both listeners should get a copy: + driver.send(fanout_target, {"context": "whatever"}, + {"method": "echo", "id": "fanout"}) + + listener1.join(timeout=30) + self.assertFalse(listener1.isAlive()) + listener2.join(timeout=30) + self.assertFalse(listener2.isAlive()) + self.assertEqual(self._broker.fanout_count, 1) + + listener1_ids = [x.message.get('id') for x in listener1.get_messages()] + listener2_ids = [x.message.get('id') for x in listener2.get_messages()] + + self.assertTrue('fanout' in listener1_ids and + 'fanout' in listener2_ids) + self.assertTrue('server1-1' in listener1_ids and + 'server1-1' not in listener2_ids) + self.assertTrue('server1-2' in listener1_ids and + 'server1-2' not in listener2_ids) + self.assertTrue('server2' in listener2_ids and + 'server2' not in listener1_ids) + if 'either-1' in listener1_ids: + self.assertTrue('either-2' in listener2_ids and + 'either-2' not in listener1_ids and + 'either-1' not in listener2_ids) + else: + self.assertTrue('either-2' in listener1_ids and + 'either-2' not in listener2_ids and + 'either-1' in listener2_ids) + driver.cleanup() + + def test_send_timeout(self): + """Verify send timeout.""" + driver = amqp_driver.ProtonDriver(self.conf, self._broker_url) + target = oslo_messaging.Target(topic="test-topic") + listener = _ListenerThread(driver.listen(target), 1) + + # the listener will drop this message: + try: + driver.send(target, + {"context": "whatever"}, + {"method": "drop"}, + wait_for_reply=True, + timeout=1.0) + except Exception as ex: + self.assertIsInstance(ex, oslo_messaging.MessagingTimeout, ex) + else: + self.assertTrue(False, "No Exception raised!") + listener.join(timeout=30) + self.assertFalse(listener.isAlive()) + driver.cleanup() + + +class TestAmqpNotification(_AmqpBrokerTestCase): + """Test sending and receiving notifications.""" + + def test_notification(self): + driver = amqp_driver.ProtonDriver(self.conf, self._broker_url) + notifications = [(oslo_messaging.Target(topic="topic-1"), 'info'), + (oslo_messaging.Target(topic="topic-1"), 'error'), + (oslo_messaging.Target(topic="topic-2"), 'debug')] + nl = driver.listen_for_notifications(notifications) + + listener = _ListenerThread(nl, 3) + targets = ['topic-1.info', + 'topic-1.bad', # should be dropped + 'bad-topic.debug', # should be dropped + 'topic-1.error', 'topic-2.debug'] + + for t in targets: + driver.send_notification(oslo_messaging.Target(topic=t), + "context", {'target': t}, + 1.0) + listener.join(timeout=30) + self.assertFalse(listener.isAlive()) + topics = [x.message.get('target') for x in listener.get_messages()] + self.assertTrue('topic-1.info' in topics) + self.assertTrue('topic-1.error' in topics) + self.assertTrue('topic-2.debug' in topics) + self.assertEqual(self._broker.dropped_count, 2) + driver.cleanup() + + +@testtools.skipUnless(pyngus, "proton modules not present") +class TestAuthentication(test_utils.BaseTestCase): + + def setUp(self): + super(TestAuthentication, self).setUp() + LOG.error("Starting Authentication Test") + # for simplicity, encode the credentials as they would appear 'on the + # wire' in a SASL frame - username and password prefixed by zero. + user_credentials = ["\0joe\0secret"] + self._broker = FakeBroker(sasl_mechanisms="PLAIN", + user_credentials=user_credentials) + self._broker.start() + + def tearDown(self): + super(TestAuthentication, self).tearDown() + self._broker.stop() + LOG.error("Authentication Test Ended") + + def test_authentication_ok(self): + """Verify that username and password given in TransportHost are + accepted by the broker. + """ + + addr = "amqp://joe:secret@%s:%d" % (self._broker.host, + self._broker.port) + url = oslo_messaging.TransportURL.parse(self.conf, addr) + driver = amqp_driver.ProtonDriver(self.conf, url) + target = oslo_messaging.Target(topic="test-topic") + listener = _ListenerThread(driver.listen(target), 1) + rc = driver.send(target, {"context": True}, + {"method": "echo"}, wait_for_reply=True) + self.assertIsNotNone(rc) + listener.join(timeout=30) + self.assertFalse(listener.isAlive()) + driver.cleanup() + + def test_authentication_failure(self): + """Verify that a bad password given in TransportHost is + rejected by the broker. + """ + + addr = "amqp://joe:badpass@%s:%d" % (self._broker.host, + self._broker.port) + url = oslo_messaging.TransportURL.parse(self.conf, addr) + driver = amqp_driver.ProtonDriver(self.conf, url) + target = oslo_messaging.Target(topic="test-topic") + _ListenerThread(driver.listen(target), 1) + self.assertRaises(oslo_messaging.MessagingTimeout, + driver.send, + target, {"context": True}, + {"method": "echo"}, + wait_for_reply=True, + timeout=2.0) + driver.cleanup() + + +@testtools.skipUnless(pyngus, "proton modules not present") +class TestFailover(test_utils.BaseTestCase): + + def setUp(self): + super(TestFailover, self).setUp() + LOG.info("Starting Failover Test") + self._brokers = [FakeBroker(), FakeBroker()] + hosts = [] + for broker in self._brokers: + hosts.append(oslo_messaging.TransportHost(hostname=broker.host, + port=broker.port)) + self._broker_url = oslo_messaging.TransportURL(self.conf, + transport="amqp", + hosts=hosts) + + def tearDown(self): + super(TestFailover, self).tearDown() + for broker in self._brokers: + if broker.isAlive(): + broker.stop() + + def test_broker_failover(self): + """Simulate failover of one broker to another.""" + self._brokers[0].start() + driver = amqp_driver.ProtonDriver(self.conf, self._broker_url) + + target = oslo_messaging.Target(topic="my-topic") + listener = _ListenerThread(driver.listen(target), 2) + + rc = driver.send(target, {"context": "whatever"}, + {"method": "echo", "id": "echo-1"}, + wait_for_reply=True, + timeout=30) + self.assertIsNotNone(rc) + self.assertEqual(rc.get('correlation-id'), 'echo-1') + # 1 request msg, 1 response: + self.assertEqual(self._brokers[0].topic_count, 1) + self.assertEqual(self._brokers[0].direct_count, 1) + + # fail broker 0 and start broker 1: + self._brokers[0].stop() + self._brokers[1].start() + deadline = time.time() + 30 + responded = False + sequence = 2 + while deadline > time.time() and not responded: + if not listener.isAlive(): + # listener may have exited after replying to an old correlation + # id: restart new listener + listener = _ListenerThread(driver.listen(target), 1) + try: + rc = driver.send(target, {"context": "whatever"}, + {"method": "echo", + "id": "echo-%d" % sequence}, + wait_for_reply=True, + timeout=2) + self.assertIsNotNone(rc) + self.assertEqual(rc.get('correlation-id'), + 'echo-%d' % sequence) + responded = True + except oslo_messaging.MessagingTimeout: + sequence += 1 + + self.assertTrue(responded) + listener.join(timeout=30) + self.assertFalse(listener.isAlive()) + + # note: stopping the broker first tests cleaning up driver without a + # connection active + self._brokers[1].stop() + driver.cleanup() + + +class FakeBroker(threading.Thread): + """A test AMQP message 'broker'.""" + + if pyngus: + class Connection(pyngus.ConnectionEventHandler): + """A single AMQP connection.""" + + def __init__(self, server, socket_, name, + sasl_mechanisms, user_credentials): + """Create a Connection using socket_.""" + self.socket = socket_ + self.name = name + self.server = server + self.connection = server.container.create_connection(name, + self) + self.connection.user_context = self + self.sasl_mechanisms = sasl_mechanisms + self.user_credentials = user_credentials + if sasl_mechanisms: + self.connection.pn_sasl.mechanisms(sasl_mechanisms) + self.connection.pn_sasl.server() + self.connection.open() + self.sender_links = set() + self.closed = False + + def destroy(self): + """Destroy the test connection.""" + while self.sender_links: + link = self.sender_links.pop() + link.destroy() + self.connection.destroy() + self.connection = None + self.socket.close() + + def fileno(self): + """Allows use of this in a select() call.""" + return self.socket.fileno() + + def process_input(self): + """Called when socket is read-ready.""" + try: + pyngus.read_socket_input(self.connection, self.socket) + except socket.error: + pass + self.connection.process(time.time()) + + def send_output(self): + """Called when socket is write-ready.""" + try: + pyngus.write_socket_output(self.connection, + self.socket) + except socket.error: + pass + self.connection.process(time.time()) + + # Pyngus ConnectionEventHandler callbacks: + + def connection_remote_closed(self, connection, reason): + """Peer has closed the connection.""" + self.connection.close() + + def connection_closed(self, connection): + """Connection close completed.""" + self.closed = True # main loop will destroy + + def connection_failed(self, connection, error): + """Connection failure detected.""" + self.connection_closed(connection) + + def sender_requested(self, connection, link_handle, + name, requested_source, properties): + """Create a new message source.""" + addr = requested_source or "source-" + uuid.uuid4().hex + link = FakeBroker.SenderLink(self.server, self, + link_handle, addr) + self.sender_links.add(link) + + def receiver_requested(self, connection, link_handle, + name, requested_target, properties): + """Create a new message consumer.""" + addr = requested_target or "target-" + uuid.uuid4().hex + FakeBroker.ReceiverLink(self.server, self, + link_handle, addr) + + def sasl_step(self, connection, pn_sasl): + if self.sasl_mechanisms == 'PLAIN': + credentials = pn_sasl.recv() + if not credentials: + return # wait until some arrives + if credentials not in self.user_credentials: + # failed + return pn_sasl.done(pn_sasl.AUTH) + pn_sasl.done(pn_sasl.OK) + + class SenderLink(pyngus.SenderEventHandler): + """An AMQP sending link.""" + def __init__(self, server, conn, handle, src_addr=None): + self.server = server + cnn = conn.connection + self.link = cnn.accept_sender(handle, + source_override=src_addr, + event_handler=self) + self.link.open() + self.routed = False + + def destroy(self): + """Destroy the link.""" + self._cleanup() + if self.link: + self.link.destroy() + self.link = None + + def send_message(self, message): + """Send a message over this link.""" + self.link.send(message) + + def _cleanup(self): + if self.routed: + self.server.remove_route(self.link.source_address, + self) + self.routed = False + + # Pyngus SenderEventHandler callbacks: + + def sender_active(self, sender_link): + self.server.add_route(self.link.source_address, self) + self.routed = True + + def sender_remote_closed(self, sender_link, error): + self._cleanup() + self.link.close() + + def sender_closed(self, sender_link): + self.destroy() + + class ReceiverLink(pyngus.ReceiverEventHandler): + """An AMQP Receiving link.""" + def __init__(self, server, conn, handle, addr=None): + self.server = server + cnn = conn.connection + self.link = cnn.accept_receiver(handle, + target_override=addr, + event_handler=self) + self.link.open() + self.link.add_capacity(10) + + # ReceiverEventHandler callbacks: + + def receiver_remote_closed(self, receiver_link, error): + self.link.close() + + def receiver_closed(self, receiver_link): + self.link.destroy() + self.link = None + + def message_received(self, receiver_link, message, handle): + """Forward this message out the proper sending link.""" + if self.server.forward_message(message): + self.link.message_accepted(handle) + else: + self.link.message_rejected(handle) + + if self.link.capacity < 1: + self.link.add_capacity(10) + + def __init__(self, server_prefix="exclusive", + broadcast_prefix="broadcast", + group_prefix="unicast", + address_separator=".", + sock_addr="", sock_port=0, + sasl_mechanisms="ANONYMOUS", + user_credentials=None): + """Create a fake broker listening on sock_addr:sock_port.""" + if not pyngus: + raise AssertionError("pyngus module not present") + threading.Thread.__init__(self) + self._server_prefix = server_prefix + address_separator + self._broadcast_prefix = broadcast_prefix + address_separator + self._group_prefix = group_prefix + address_separator + self._address_separator = address_separator + self._sasl_mechanisms = sasl_mechanisms + self._user_credentials = user_credentials + self._wakeup_pipe = os.pipe() + self._my_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._my_socket.bind((sock_addr, sock_port)) + self.host, self.port = self._my_socket.getsockname() + self.container = pyngus.Container("test_server_%s:%d" + % (self.host, self.port)) + self._connections = {} + self._sources = {} + # count of messages forwarded, by messaging pattern + self.direct_count = 0 + self.topic_count = 0 + self.fanout_count = 0 + self.dropped_count = 0 + + def start(self): + """Start the server.""" + LOG.info("Starting Test Broker on %s:%d", self.host, self.port) + self._shutdown = False + self.daemon = True + self._my_socket.listen(10) + super(FakeBroker, self).start() + + def stop(self): + """Shutdown the server.""" + LOG.info("Stopping test Broker %s:%d", self.host, self.port) + self._shutdown = True + os.write(self._wakeup_pipe[1], "!") + self.join() + LOG.info("Test Broker %s:%d stopped", self.host, self.port) + + def run(self): + """Process I/O and timer events until the broker is stopped.""" + LOG.info("Test Broker on %s:%d started", self.host, self.port) + while not self._shutdown: + readers, writers, timers = self.container.need_processing() + + # map pyngus Connections back to _TestConnections: + readfd = [c.user_context for c in readers] + readfd.extend([self._my_socket, self._wakeup_pipe[0]]) + writefd = [c.user_context for c in writers] + + timeout = None + if timers: + # [0] == next expiring timer + deadline = timers[0].next_tick + now = time.time() + timeout = 0 if deadline <= now else deadline - now + + readable, writable, ignore = select.select(readfd, + writefd, + [], + timeout) + worked = set() + for r in readable: + if r is self._my_socket: + # new inbound connection request received, + # create a new Connection for it: + client_socket, client_address = self._my_socket.accept() + name = str(client_address) + conn = FakeBroker.Connection(self, client_socket, name, + self._sasl_mechanisms, + self._user_credentials) + self._connections[conn.name] = conn + elif r is self._wakeup_pipe[0]: + os.read(self._wakeup_pipe[0], 512) + else: + r.process_input() + worked.add(r) + + for t in timers: + now = time.time() + if t.next_tick > now: + break + t.process(now) + conn = t.user_context + worked.add(conn) + + for w in writable: + w.send_output() + worked.add(w) + + # clean up any closed connections: + while worked: + conn = worked.pop() + if conn.closed: + del self._connections[conn.name] + conn.destroy() + + # Shutting down + self._my_socket.close() + for conn in self._connections.itervalues(): + conn.destroy() + return 0 + + def add_route(self, address, link): + # route from address -> link[, link ...] + if address not in self._sources: + self._sources[address] = [link] + elif link not in self._sources[address]: + self._sources[address].append(link) + + def remove_route(self, address, link): + if address in self._sources: + if link in self._sources[address]: + self._sources[address].remove(link) + if not self._sources[address]: + del self._sources[address] + + def forward_message(self, message): + # returns True if message was routed + dest = message.address + if dest not in self._sources: + self.dropped_count += 1 + return False + LOG.debug("Forwarding [%s]", dest) + # route "behavior" determined by prefix: + if dest.startswith(self._broadcast_prefix): + self.fanout_count += 1 + for link in self._sources[dest]: + LOG.debug("Broadcast to %s", dest) + link.send_message(message) + elif dest.startswith(self._group_prefix): + # round-robin: + self.topic_count += 1 + link = self._sources[dest].pop(0) + link.send_message(message) + LOG.debug("Send to %s", dest) + self._sources[dest].append(link) + else: + # unicast: + self.direct_count += 1 + LOG.debug("Unicast to %s", dest) + self._sources[dest][0].send_message(message) + return True diff --git a/oslo_messaging/tests/test_exception_serialization.py b/oslo_messaging/tests/test_exception_serialization.py new file mode 100644 index 000000000..1d1de3173 --- /dev/null +++ b/oslo_messaging/tests/test_exception_serialization.py @@ -0,0 +1,307 @@ + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys + +import six +import testscenarios + +from oslo.serialization import jsonutils +import oslo_messaging +from oslo_messaging._drivers import common as exceptions +from oslo_messaging.tests import utils as test_utils + +load_tests = testscenarios.load_tests_apply_scenarios + +EXCEPTIONS_MODULE = 'exceptions' if six.PY2 else 'builtins' + + +class NovaStyleException(Exception): + + format = 'I am Nova' + + def __init__(self, message=None, **kwargs): + self.kwargs = kwargs + if not message: + message = self.format % kwargs + super(NovaStyleException, self).__init__(message) + + +class KwargsStyleException(NovaStyleException): + + format = 'I am %(who)s' + + +def add_remote_postfix(ex): + ex_type = type(ex) + message = str(ex) + str_override = lambda self: message + new_ex_type = type(ex_type.__name__ + "_Remote", (ex_type,), + {'__str__': str_override, + '__unicode__': str_override}) + new_ex_type.__module__ = '%s_Remote' % ex.__class__.__module__ + try: + ex.__class__ = new_ex_type + except TypeError: + ex.args = (message,) + ex.args[1:] + return ex + + +class SerializeRemoteExceptionTestCase(test_utils.BaseTestCase): + + _log_failure = [ + ('log_failure', dict(log_failure=True)), + ('do_not_log_failure', dict(log_failure=False)), + ] + + _add_remote = [ + ('add_remote', dict(add_remote=True)), + ('do_not_add_remote', dict(add_remote=False)), + ] + + _exception_types = [ + ('bog_standard', dict(cls=Exception, + args=['test'], + kwargs={}, + clsname='Exception', + modname=EXCEPTIONS_MODULE, + msg='test')), + ('nova_style', dict(cls=NovaStyleException, + args=[], + kwargs={}, + clsname='NovaStyleException', + modname=__name__, + msg='I am Nova')), + ('nova_style_with_msg', dict(cls=NovaStyleException, + args=['testing'], + kwargs={}, + clsname='NovaStyleException', + modname=__name__, + msg='testing')), + ('kwargs_style', dict(cls=KwargsStyleException, + args=[], + kwargs={'who': 'Oslo'}, + clsname='KwargsStyleException', + modname=__name__, + msg='I am Oslo')), + ] + + @classmethod + def generate_scenarios(cls): + cls.scenarios = testscenarios.multiply_scenarios(cls._log_failure, + cls._add_remote, + cls._exception_types) + + def setUp(self): + super(SerializeRemoteExceptionTestCase, self).setUp() + + def test_serialize_remote_exception(self): + errors = [] + + def stub_error(msg, *a, **kw): + if (a and len(a) == 1 and isinstance(a[0], dict) and a[0]): + a = a[0] + errors.append(str(msg) % a) + + self.stubs.Set(exceptions.LOG, 'error', stub_error) + + try: + try: + raise self.cls(*self.args, **self.kwargs) + except Exception as ex: + cls_error = ex + if self.add_remote: + ex = add_remote_postfix(ex) + raise ex + except Exception: + exc_info = sys.exc_info() + + serialized = exceptions.serialize_remote_exception( + exc_info, log_failure=self.log_failure) + + failure = jsonutils.loads(serialized) + + self.assertEqual(self.clsname, failure['class'], failure) + self.assertEqual(self.modname, failure['module']) + self.assertEqual(self.msg, failure['message']) + self.assertEqual([self.msg], failure['args']) + self.assertEqual(self.kwargs, failure['kwargs']) + + # Note: _Remote prefix not stripped from tracebacks + tb = cls_error.__class__.__name__ + ': ' + self.msg + self.assertIn(tb, ''.join(failure['tb'])) + + if self.log_failure: + self.assertTrue(len(errors) > 0, errors) + else: + self.assertEqual(0, len(errors), errors) + + +SerializeRemoteExceptionTestCase.generate_scenarios() + + +class DeserializeRemoteExceptionTestCase(test_utils.BaseTestCase): + + _standard_allowed = [__name__] + + scenarios = [ + ('bog_standard', + dict(allowed=_standard_allowed, + clsname='Exception', + modname=EXCEPTIONS_MODULE, + cls=Exception, + args=['test'], + kwargs={}, + str='test\ntraceback\ntraceback\n', + remote_name='Exception', + remote_args=('test\ntraceback\ntraceback\n', ), + remote_kwargs={})), + ('nova_style', + dict(allowed=_standard_allowed, + clsname='NovaStyleException', + modname=__name__, + cls=NovaStyleException, + args=[], + kwargs={}, + str='test\ntraceback\ntraceback\n', + remote_name='NovaStyleException_Remote', + remote_args=('I am Nova', ), + remote_kwargs={})), + ('nova_style_with_msg', + dict(allowed=_standard_allowed, + clsname='NovaStyleException', + modname=__name__, + cls=NovaStyleException, + args=['testing'], + kwargs={}, + str='test\ntraceback\ntraceback\n', + remote_name='NovaStyleException_Remote', + remote_args=('testing', ), + remote_kwargs={})), + ('kwargs_style', + dict(allowed=_standard_allowed, + clsname='KwargsStyleException', + modname=__name__, + cls=KwargsStyleException, + args=[], + kwargs={'who': 'Oslo'}, + str='test\ntraceback\ntraceback\n', + remote_name='KwargsStyleException_Remote', + remote_args=('I am Oslo', ), + remote_kwargs={})), + ('not_allowed', + dict(allowed=[], + clsname='NovaStyleException', + modname=__name__, + cls=oslo_messaging.RemoteError, + args=[], + kwargs={}, + str=("Remote error: NovaStyleException test\n" + "[%r]." % u'traceback\ntraceback\n'), + msg=("Remote error: NovaStyleException test\n" + "[%r]." % u'traceback\ntraceback\n'), + remote_name='RemoteError', + remote_args=(), + remote_kwargs={'exc_type': 'NovaStyleException', + 'value': 'test', + 'traceback': 'traceback\ntraceback\n'})), + ('unknown_module', + dict(allowed=['notexist'], + clsname='Exception', + modname='notexist', + cls=oslo_messaging.RemoteError, + args=[], + kwargs={}, + str=("Remote error: Exception test\n" + "[%r]." % u'traceback\ntraceback\n'), + msg=("Remote error: Exception test\n" + "[%r]." % u'traceback\ntraceback\n'), + remote_name='RemoteError', + remote_args=(), + remote_kwargs={'exc_type': 'Exception', + 'value': 'test', + 'traceback': 'traceback\ntraceback\n'})), + ('unknown_exception', + dict(allowed=[], + clsname='FarcicalError', + modname=EXCEPTIONS_MODULE, + cls=oslo_messaging.RemoteError, + args=[], + kwargs={}, + str=("Remote error: FarcicalError test\n" + "[%r]." % u'traceback\ntraceback\n'), + msg=("Remote error: FarcicalError test\n" + "[%r]." % u'traceback\ntraceback\n'), + remote_name='RemoteError', + remote_args=(), + remote_kwargs={'exc_type': 'FarcicalError', + 'value': 'test', + 'traceback': 'traceback\ntraceback\n'})), + ('unknown_kwarg', + dict(allowed=[], + clsname='Exception', + modname=EXCEPTIONS_MODULE, + cls=oslo_messaging.RemoteError, + args=[], + kwargs={'foobar': 'blaa'}, + str=("Remote error: Exception test\n" + "[%r]." % u'traceback\ntraceback\n'), + msg=("Remote error: Exception test\n" + "[%r]." % u'traceback\ntraceback\n'), + remote_name='RemoteError', + remote_args=(), + remote_kwargs={'exc_type': 'Exception', + 'value': 'test', + 'traceback': 'traceback\ntraceback\n'})), + ('system_exit', + dict(allowed=[], + clsname='SystemExit', + modname=EXCEPTIONS_MODULE, + cls=oslo_messaging.RemoteError, + args=[], + kwargs={}, + str=("Remote error: SystemExit test\n" + "[%r]." % u'traceback\ntraceback\n'), + msg=("Remote error: SystemExit test\n" + "[%r]." % u'traceback\ntraceback\n'), + remote_name='RemoteError', + remote_args=(), + remote_kwargs={'exc_type': 'SystemExit', + 'value': 'test', + 'traceback': 'traceback\ntraceback\n'})), + ] + + def test_deserialize_remote_exception(self): + failure = { + 'class': self.clsname, + 'module': self.modname, + 'message': 'test', + 'tb': ['traceback\ntraceback\n'], + 'args': self.args, + 'kwargs': self.kwargs, + } + + serialized = jsonutils.dumps(failure) + + ex = exceptions.deserialize_remote_exception(serialized, self.allowed) + + self.assertIsInstance(ex, self.cls) + self.assertEqual(self.remote_name, ex.__class__.__name__) + self.assertEqual(self.str, six.text_type(ex)) + if hasattr(self, 'msg'): + self.assertEqual(self.msg, six.text_type(ex)) + self.assertEqual((self.msg,) + self.remote_args, ex.args) + else: + self.assertEqual(self.remote_args, ex.args) diff --git a/oslo_messaging/tests/test_expected_exceptions.py b/oslo_messaging/tests/test_expected_exceptions.py new file mode 100644 index 000000000..40c4a22fb --- /dev/null +++ b/oslo_messaging/tests/test_expected_exceptions.py @@ -0,0 +1,66 @@ + +# Copyright 2012 OpenStack Foundation +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import oslo_messaging +from oslo_messaging.tests import utils as test_utils + + +class TestExpectedExceptions(test_utils.BaseTestCase): + + def test_exception(self): + e = None + try: + try: + raise ValueError() + except Exception: + raise oslo_messaging.ExpectedException() + except oslo_messaging.ExpectedException as e: + self.assertIsInstance(e, oslo_messaging.ExpectedException) + self.assertTrue(hasattr(e, 'exc_info')) + self.assertIsInstance(e.exc_info[1], ValueError) + + def test_decorator_expected(self): + class FooException(Exception): + pass + + @oslo_messaging.expected_exceptions(FooException) + def naughty(): + raise FooException() + + self.assertRaises(oslo_messaging.ExpectedException, naughty) + + def test_decorator_expected_subclass(self): + class FooException(Exception): + pass + + class BarException(FooException): + pass + + @oslo_messaging.expected_exceptions(FooException) + def naughty(): + raise BarException() + + self.assertRaises(oslo_messaging.ExpectedException, naughty) + + def test_decorator_unexpected(self): + class FooException(Exception): + pass + + @oslo_messaging.expected_exceptions(FooException) + def really_naughty(): + raise ValueError() + + self.assertRaises(ValueError, really_naughty) diff --git a/tests/test_opts.py b/oslo_messaging/tests/test_opts.py similarity index 95% rename from tests/test_opts.py rename to oslo_messaging/tests/test_opts.py index 920fa612d..37b621f6d 100644 --- a/tests/test_opts.py +++ b/oslo_messaging/tests/test_opts.py @@ -16,10 +16,10 @@ import pkg_resources import testtools try: - from oslo.messaging import opts + from oslo_messaging import opts except ImportError: opts = None -from tests import utils as test_utils +from oslo_messaging.tests import utils as test_utils class OptsTestCase(test_utils.BaseTestCase): diff --git a/oslo_messaging/tests/test_target.py b/oslo_messaging/tests/test_target.py new file mode 100644 index 000000000..049f4f768 --- /dev/null +++ b/oslo_messaging/tests/test_target.py @@ -0,0 +1,177 @@ + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import testscenarios + +import oslo_messaging +from oslo_messaging.tests import utils as test_utils + +load_tests = testscenarios.load_tests_apply_scenarios + + +class TargetConstructorTestCase(test_utils.BaseTestCase): + + scenarios = [ + ('all_none', dict(kwargs=dict())), + ('exchange', dict(kwargs=dict(exchange='testexchange'))), + ('topic', dict(kwargs=dict(topic='testtopic'))), + ('namespace', dict(kwargs=dict(namespace='testnamespace'))), + ('version', dict(kwargs=dict(version='3.4'))), + ('server', dict(kwargs=dict(server='testserver'))), + ('fanout', dict(kwargs=dict(fanout=True))), + ] + + def test_constructor(self): + target = oslo_messaging.Target(**self.kwargs) + for k in self.kwargs: + self.assertEqual(self.kwargs[k], getattr(target, k)) + for k in ['exchange', 'topic', 'namespace', + 'version', 'server', 'fanout']: + if k in self.kwargs: + continue + self.assertIsNone(getattr(target, k)) + + +class TargetCallableTestCase(test_utils.BaseTestCase): + + scenarios = [ + ('all_none', dict(attrs=dict(), kwargs=dict(), vals=dict())), + ('exchange_attr', dict(attrs=dict(exchange='testexchange'), + kwargs=dict(), + vals=dict(exchange='testexchange'))), + ('exchange_arg', dict(attrs=dict(), + kwargs=dict(exchange='testexchange'), + vals=dict(exchange='testexchange'))), + ('topic_attr', dict(attrs=dict(topic='testtopic'), + kwargs=dict(), + vals=dict(topic='testtopic'))), + ('topic_arg', dict(attrs=dict(), + kwargs=dict(topic='testtopic'), + vals=dict(topic='testtopic'))), + ('namespace_attr', dict(attrs=dict(namespace='testnamespace'), + kwargs=dict(), + vals=dict(namespace='testnamespace'))), + ('namespace_arg', dict(attrs=dict(), + kwargs=dict(namespace='testnamespace'), + vals=dict(namespace='testnamespace'))), + ('version_attr', dict(attrs=dict(version='3.4'), + kwargs=dict(), + vals=dict(version='3.4'))), + ('version_arg', dict(attrs=dict(), + kwargs=dict(version='3.4'), + vals=dict(version='3.4'))), + ('server_attr', dict(attrs=dict(server='testserver'), + kwargs=dict(), + vals=dict(server='testserver'))), + ('server_arg', dict(attrs=dict(), + kwargs=dict(server='testserver'), + vals=dict(server='testserver'))), + ('fanout_attr', dict(attrs=dict(fanout=True), + kwargs=dict(), + vals=dict(fanout=True))), + ('fanout_arg', dict(attrs=dict(), + kwargs=dict(fanout=True), + vals=dict(fanout=True))), + ] + + def test_callable(self): + target = oslo_messaging.Target(**self.attrs) + target = target(**self.kwargs) + for k in self.vals: + self.assertEqual(self.vals[k], getattr(target, k)) + for k in ['exchange', 'topic', 'namespace', + 'version', 'server', 'fanout']: + if k in self.vals: + continue + self.assertIsNone(getattr(target, k)) + + +class TargetReprTestCase(test_utils.BaseTestCase): + + scenarios = [ + ('all_none', dict(kwargs=dict(), repr='')), + ('exchange', dict(kwargs=dict(exchange='testexchange'), + repr='exchange=testexchange')), + ('topic', dict(kwargs=dict(topic='testtopic'), + repr='topic=testtopic')), + ('namespace', dict(kwargs=dict(namespace='testnamespace'), + repr='namespace=testnamespace')), + ('version', dict(kwargs=dict(version='3.4'), + repr='version=3.4')), + ('server', dict(kwargs=dict(server='testserver'), + repr='server=testserver')), + ('fanout', dict(kwargs=dict(fanout=True), + repr='fanout=True')), + ('exchange_and_fanout', dict(kwargs=dict(exchange='testexchange', + fanout=True), + repr='exchange=testexchange, ' + 'fanout=True')), + ] + + def test_repr(self): + target = oslo_messaging.Target(**self.kwargs) + self.assertEqual('', str(target)) + + +_notset = object() + + +class EqualityTestCase(test_utils.BaseTestCase): + + @classmethod + def generate_scenarios(cls): + attr = [ + ('exchange', dict(attr='exchange')), + ('topic', dict(attr='topic')), + ('namespace', dict(attr='namespace')), + ('version', dict(attr='version')), + ('server', dict(attr='server')), + ('fanout', dict(attr='fanout')), + ] + a = [ + ('a_notset', dict(a_value=_notset)), + ('a_none', dict(a_value=None)), + ('a_empty', dict(a_value='')), + ('a_foo', dict(a_value='foo')), + ('a_bar', dict(a_value='bar')), + ] + b = [ + ('b_notset', dict(b_value=_notset)), + ('b_none', dict(b_value=None)), + ('b_empty', dict(b_value='')), + ('b_foo', dict(b_value='foo')), + ('b_bar', dict(b_value='bar')), + ] + + cls.scenarios = testscenarios.multiply_scenarios(attr, a, b) + for s in cls.scenarios: + s[1]['equals'] = (s[1]['a_value'] == s[1]['b_value']) + + def test_equality(self): + a_kwargs = {self.attr: self.a_value} + b_kwargs = {self.attr: self.b_value} + + a = oslo_messaging.Target(**a_kwargs) + b = oslo_messaging.Target(**b_kwargs) + + if self.equals: + self.assertEqual(a, b) + self.assertFalse(a != b) + else: + self.assertNotEqual(a, b) + self.assertFalse(a == b) + + +EqualityTestCase.generate_scenarios() diff --git a/oslo_messaging/tests/test_transport.py b/oslo_messaging/tests/test_transport.py new file mode 100644 index 000000000..53d991a88 --- /dev/null +++ b/oslo_messaging/tests/test_transport.py @@ -0,0 +1,367 @@ + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import fixtures +import six +from six.moves import mox +from stevedore import driver +import testscenarios + +from oslo.config import cfg +import oslo_messaging +from oslo_messaging.tests import utils as test_utils +from oslo_messaging import transport + +load_tests = testscenarios.load_tests_apply_scenarios + + +class _FakeDriver(object): + + def __init__(self, conf): + self.conf = conf + + def send(self, *args, **kwargs): + pass + + def send_notification(self, *args, **kwargs): + pass + + def listen(self, target): + pass + + +class _FakeManager(object): + + def __init__(self, driver): + self.driver = driver + + +class GetTransportTestCase(test_utils.BaseTestCase): + + scenarios = [ + ('rpc_backend', + dict(url=None, transport_url=None, rpc_backend='testbackend', + control_exchange=None, allowed=None, aliases=None, + expect=dict(backend='testbackend', + exchange=None, + url='testbackend:', + allowed=[]))), + ('transport_url', + dict(url=None, transport_url='testtransport:', rpc_backend=None, + control_exchange=None, allowed=None, aliases=None, + expect=dict(backend='testtransport', + exchange=None, + url='testtransport:', + allowed=[]))), + ('url_param', + dict(url='testtransport:', transport_url=None, rpc_backend=None, + control_exchange=None, allowed=None, aliases=None, + expect=dict(backend='testtransport', + exchange=None, + url='testtransport:', + allowed=[]))), + ('control_exchange', + dict(url=None, transport_url=None, rpc_backend='testbackend', + control_exchange='testexchange', allowed=None, aliases=None, + expect=dict(backend='testbackend', + exchange='testexchange', + url='testbackend:', + allowed=[]))), + ('allowed_remote_exmods', + dict(url=None, transport_url=None, rpc_backend='testbackend', + control_exchange=None, allowed=['foo', 'bar'], aliases=None, + expect=dict(backend='testbackend', + exchange=None, + url='testbackend:', + allowed=['foo', 'bar']))), + ('rpc_backend_aliased', + dict(url=None, transport_url=None, rpc_backend='testfoo', + control_exchange=None, allowed=None, + aliases=dict(testfoo='testbackend'), + expect=dict(backend='testbackend', + exchange=None, + url='testbackend:', + allowed=[]))), + ('transport_url_aliased', + dict(url=None, transport_url='testfoo:', rpc_backend=None, + control_exchange=None, allowed=None, + aliases=dict(testfoo='testtransport'), + expect=dict(backend='testtransport', + exchange=None, + url='testtransport:', + allowed=[]))), + ('url_param_aliased', + dict(url='testfoo:', transport_url=None, rpc_backend=None, + control_exchange=None, allowed=None, + aliases=dict(testfoo='testtransport'), + expect=dict(backend='testtransport', + exchange=None, + url='testtransport:', + allowed=[]))), + ] + + def test_get_transport(self): + self.config(rpc_backend=self.rpc_backend, + control_exchange=self.control_exchange, + transport_url=self.transport_url) + + self.mox.StubOutWithMock(driver, 'DriverManager') + + invoke_args = [self.conf, + oslo_messaging.TransportURL.parse(self.conf, + self.expect['url'])] + invoke_kwds = dict(default_exchange=self.expect['exchange'], + allowed_remote_exmods=self.expect['allowed']) + + drvr = _FakeDriver(self.conf) + driver.DriverManager('oslo.messaging.drivers', + self.expect['backend'], + invoke_on_load=True, + invoke_args=invoke_args, + invoke_kwds=invoke_kwds).\ + AndReturn(_FakeManager(drvr)) + + self.mox.ReplayAll() + + kwargs = dict(url=self.url) + if self.allowed is not None: + kwargs['allowed_remote_exmods'] = self.allowed + if self.aliases is not None: + kwargs['aliases'] = self.aliases + transport_ = oslo_messaging.get_transport(self.conf, **kwargs) + + self.assertIsNotNone(transport_) + self.assertIs(transport_.conf, self.conf) + self.assertIs(transport_._driver, drvr) + + +class GetTransportSadPathTestCase(test_utils.BaseTestCase): + + scenarios = [ + ('invalid_transport_url', + dict(url=None, transport_url='invalid', rpc_backend=None, + ex=dict(cls=oslo_messaging.InvalidTransportURL, + msg_contains='No scheme specified', + url='invalid'))), + ('invalid_url_param', + dict(url='invalid', transport_url=None, rpc_backend=None, + ex=dict(cls=oslo_messaging.InvalidTransportURL, + msg_contains='No scheme specified', + url='invalid'))), + ('driver_load_failure', + dict(url=None, transport_url=None, rpc_backend='testbackend', + ex=dict(cls=oslo_messaging.DriverLoadFailure, + msg_contains='Failed to load', + driver='testbackend'))), + ] + + def test_get_transport_sad(self): + self.config(rpc_backend=self.rpc_backend, + transport_url=self.transport_url) + + if self.rpc_backend: + self.mox.StubOutWithMock(driver, 'DriverManager') + + invoke_args = [self.conf, + oslo_messaging.TransportURL.parse(self.conf, + self.url)] + invoke_kwds = dict(default_exchange='openstack', + allowed_remote_exmods=[]) + + driver.DriverManager('oslo.messaging.drivers', + self.rpc_backend, + invoke_on_load=True, + invoke_args=invoke_args, + invoke_kwds=invoke_kwds).\ + AndRaise(RuntimeError()) + + self.mox.ReplayAll() + + try: + oslo_messaging.get_transport(self.conf, url=self.url) + self.assertFalse(True) + except Exception as ex: + ex_cls = self.ex.pop('cls') + ex_msg_contains = self.ex.pop('msg_contains') + + self.assertIsInstance(ex, oslo_messaging.MessagingException) + self.assertIsInstance(ex, ex_cls) + self.assertIn(ex_msg_contains, six.text_type(ex)) + + for k, v in self.ex.items(): + self.assertTrue(hasattr(ex, k)) + self.assertEqual(v, str(getattr(ex, k))) + + +# FIXME(markmc): this could be used elsewhere +class _SetDefaultsFixture(fixtures.Fixture): + + def __init__(self, set_defaults, opts, *names): + super(_SetDefaultsFixture, self).__init__() + self.set_defaults = set_defaults + self.opts = opts + self.names = names + + def setUp(self): + super(_SetDefaultsFixture, self).setUp() + + # FIXME(markmc): this comes from Id5c1f3ba + def first(seq, default=None, key=None): + if key is None: + key = bool + return next(six.moves.filter(key, seq), default) + + def default(opts, name): + return first(opts, key=lambda o: o.name == name).default + + orig_defaults = {} + for n in self.names: + orig_defaults[n] = default(self.opts, n) + + def restore_defaults(): + self.set_defaults(**orig_defaults) + + self.addCleanup(restore_defaults) + + +class TestSetDefaults(test_utils.BaseTestCase): + + def setUp(self): + super(TestSetDefaults, self).setUp(conf=cfg.ConfigOpts()) + self.useFixture(_SetDefaultsFixture( + oslo_messaging.set_transport_defaults, + transport._transport_opts, + 'control_exchange')) + + def test_set_default_control_exchange(self): + oslo_messaging.set_transport_defaults(control_exchange='foo') + + self.mox.StubOutWithMock(driver, 'DriverManager') + invoke_kwds = mox.ContainsKeyValue('default_exchange', 'foo') + driver.DriverManager(mox.IgnoreArg(), + mox.IgnoreArg(), + invoke_on_load=mox.IgnoreArg(), + invoke_args=mox.IgnoreArg(), + invoke_kwds=invoke_kwds).\ + AndReturn(_FakeManager(_FakeDriver(self.conf))) + self.mox.ReplayAll() + + oslo_messaging.get_transport(self.conf) + + +class TestTransportMethodArgs(test_utils.BaseTestCase): + + _target = oslo_messaging.Target(topic='topic', server='server') + + def test_send_defaults(self): + t = transport.Transport(_FakeDriver(cfg.CONF)) + + self.mox.StubOutWithMock(t._driver, 'send') + t._driver.send(self._target, 'ctxt', 'message', + wait_for_reply=None, + timeout=None, retry=None) + self.mox.ReplayAll() + + t._send(self._target, 'ctxt', 'message') + + def test_send_all_args(self): + t = transport.Transport(_FakeDriver(cfg.CONF)) + + self.mox.StubOutWithMock(t._driver, 'send') + t._driver.send(self._target, 'ctxt', 'message', + wait_for_reply='wait_for_reply', + timeout='timeout', retry='retry') + self.mox.ReplayAll() + + t._send(self._target, 'ctxt', 'message', + wait_for_reply='wait_for_reply', + timeout='timeout', retry='retry') + + def test_send_notification(self): + t = transport.Transport(_FakeDriver(cfg.CONF)) + + self.mox.StubOutWithMock(t._driver, 'send_notification') + t._driver.send_notification(self._target, 'ctxt', 'message', 1.0, + retry=None) + self.mox.ReplayAll() + + t._send_notification(self._target, 'ctxt', 'message', version=1.0) + + def test_send_notification_all_args(self): + t = transport.Transport(_FakeDriver(cfg.CONF)) + + self.mox.StubOutWithMock(t._driver, 'send_notification') + t._driver.send_notification(self._target, 'ctxt', 'message', 1.0, + retry=5) + self.mox.ReplayAll() + + t._send_notification(self._target, 'ctxt', 'message', version=1.0, + retry=5) + + def test_listen(self): + t = transport.Transport(_FakeDriver(cfg.CONF)) + + self.mox.StubOutWithMock(t._driver, 'listen') + t._driver.listen(self._target) + self.mox.ReplayAll() + + t._listen(self._target) + + +class TestTransportUrlCustomisation(test_utils.BaseTestCase): + def setUp(self): + super(TestTransportUrlCustomisation, self).setUp() + self.url1 = transport.TransportURL.parse(self.conf, "fake://vhost1") + self.url2 = transport.TransportURL.parse(self.conf, "fake://vhost2") + self.url3 = transport.TransportURL.parse(self.conf, "fake://vhost1") + + def test_hash(self): + urls = {} + urls[self.url1] = self.url1 + urls[self.url2] = self.url2 + urls[self.url3] = self.url3 + self.assertEqual(2, len(urls)) + + def test_eq(self): + self.assertEqual(self.url1, self.url3) + self.assertNotEqual(self.url1, self.url2) + + +class TestTransportHostCustomisation(test_utils.BaseTestCase): + def setUp(self): + super(TestTransportHostCustomisation, self).setUp() + self.host1 = transport.TransportHost("host1", 5662, "user", "pass") + self.host2 = transport.TransportHost("host1", 5662, "user", "pass") + self.host3 = transport.TransportHost("host1", 5663, "user", "pass") + self.host4 = transport.TransportHost("host1", 5662, "user2", "pass") + self.host5 = transport.TransportHost("host1", 5662, "user", "pass2") + self.host6 = transport.TransportHost("host2", 5662, "user", "pass") + + def test_hash(self): + hosts = {} + hosts[self.host1] = self.host1 + hosts[self.host2] = self.host2 + hosts[self.host3] = self.host3 + hosts[self.host4] = self.host4 + hosts[self.host5] = self.host5 + hosts[self.host6] = self.host6 + self.assertEqual(5, len(hosts)) + + def test_eq(self): + self.assertEqual(self.host1, self.host2) + self.assertNotEqual(self.host1, self.host3) + self.assertNotEqual(self.host1, self.host4) + self.assertNotEqual(self.host1, self.host5) + self.assertNotEqual(self.host1, self.host6) diff --git a/oslo_messaging/tests/test_urls.py b/oslo_messaging/tests/test_urls.py new file mode 100644 index 000000000..19f9c923b --- /dev/null +++ b/oslo_messaging/tests/test_urls.py @@ -0,0 +1,237 @@ + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import testscenarios + +import oslo_messaging +from oslo_messaging.tests import utils as test_utils + +load_tests = testscenarios.load_tests_apply_scenarios + + +class TestParseURL(test_utils.BaseTestCase): + + scenarios = [ + ('transport', + dict(url='foo:', aliases=None, + expect=dict(transport='foo'))), + ('transport_aliased', + dict(url='bar:', aliases=dict(bar='foo'), + expect=dict(transport='foo'))), + ('virtual_host_slash', + dict(url='foo:////', aliases=None, + expect=dict(transport='foo', virtual_host='/'))), + ('virtual_host', + dict(url='foo:///bar', aliases=None, + expect=dict(transport='foo', virtual_host='bar'))), + ('host', + dict(url='foo://host/bar', aliases=None, + expect=dict(transport='foo', + virtual_host='bar', + hosts=[ + dict(host='host'), + ]))), + ('ipv6_host', + dict(url='foo://[ffff::1]/bar', aliases=None, + expect=dict(transport='foo', + virtual_host='bar', + hosts=[ + dict(host='ffff::1'), + ]))), + ('port', + dict(url='foo://host:1234/bar', aliases=None, + expect=dict(transport='foo', + virtual_host='bar', + hosts=[ + dict(host='host', port=1234), + ]))), + ('ipv6_port', + dict(url='foo://[ffff::1]:1234/bar', aliases=None, + expect=dict(transport='foo', + virtual_host='bar', + hosts=[ + dict(host='ffff::1', port=1234), + ]))), + ('username', + dict(url='foo://u@host:1234/bar', aliases=None, + expect=dict(transport='foo', + virtual_host='bar', + hosts=[ + dict(host='host', port=1234, username='u'), + ]))), + ('password', + dict(url='foo://u:p@host:1234/bar', aliases=None, + expect=dict(transport='foo', + virtual_host='bar', + hosts=[ + dict(host='host', port=1234, + username='u', password='p'), + ]))), + ('creds_no_host', + dict(url='foo://u:p@/bar', aliases=None, + expect=dict(transport='foo', + virtual_host='bar', + hosts=[ + dict(username='u', password='p'), + ]))), + ('multi_host', + dict(url='foo://u:p@host1:1234,host2:4321/bar', aliases=None, + expect=dict(transport='foo', + virtual_host='bar', + hosts=[ + dict(host='host1', port=1234, + username='u', password='p'), + dict(host='host2', port=4321), + ]))), + ('multi_creds', + dict(url='foo://u1:p1@host1:1234,u2:p2@host2:4321/bar', aliases=None, + expect=dict(transport='foo', + virtual_host='bar', + hosts=[ + dict(host='host1', port=1234, + username='u1', password='p1'), + dict(host='host2', port=4321, + username='u2', password='p2'), + ]))), + ('multi_creds_ipv6', + dict(url='foo://u1:p1@[ffff::1]:1234,u2:p2@[ffff::2]:4321/bar', + aliases=None, + expect=dict(transport='foo', + virtual_host='bar', + hosts=[ + dict(host='ffff::1', port=1234, + username='u1', password='p1'), + dict(host='ffff::2', port=4321, + username='u2', password='p2'), + ]))), + ] + + def test_parse_url(self): + self.config(rpc_backend=None) + + url = oslo_messaging.TransportURL.parse(self.conf, self.url, + self.aliases) + + hosts = [] + for host in self.expect.get('hosts', []): + hosts.append(oslo_messaging.TransportHost(host.get('host'), + host.get('port'), + host.get('username'), + host.get('password'))) + expected = oslo_messaging.TransportURL(self.conf, + self.expect.get('transport'), + self.expect.get('virtual_host'), + hosts) + + self.assertEqual(expected, url) + + +class TestFormatURL(test_utils.BaseTestCase): + + scenarios = [ + ('rpc_backend', + dict(rpc_backend='testbackend', + transport=None, + virtual_host=None, + hosts=[], + aliases=None, + expected='testbackend:///')), + ('rpc_backend_aliased', + dict(rpc_backend='testfoo', + transport=None, + virtual_host=None, + hosts=[], + aliases=dict(testfoo='testbackend'), + expected='testbackend:///')), + ('transport', + dict(rpc_backend=None, + transport='testtransport', + virtual_host=None, + hosts=[], + aliases=None, + expected='testtransport:///')), + ('transport_aliased', + dict(rpc_backend=None, + transport='testfoo', + virtual_host=None, + hosts=[], + aliases=dict(testfoo='testtransport'), + expected='testtransport:///')), + ('virtual_host', + dict(rpc_backend=None, + transport='testtransport', + virtual_host='/vhost', + hosts=[], + aliases=None, + expected='testtransport:////vhost')), + ('host', + dict(rpc_backend=None, + transport='testtransport', + virtual_host='/', + hosts=[ + dict(hostname='host', + port=10, + username='bob', + password='secret'), + ], + aliases=None, + expected='testtransport://bob:secret@host:10//')), + ('multi_host', + dict(rpc_backend=None, + transport='testtransport', + virtual_host='', + hosts=[ + dict(hostname='h1', + port=1000, + username='b1', + password='s1'), + dict(hostname='h2', + port=2000, + username='b2', + password='s2'), + ], + aliases=None, + expected='testtransport://b1:s1@h1:1000,b2:s2@h2:2000/')), + ('quoting', + dict(rpc_backend=None, + transport='testtransport', + virtual_host='/$', + hosts=[ + dict(hostname='host', + port=10, + username='b$', + password='s&'), + ], + aliases=None, + expected='testtransport://b%24:s%26@host:10//%24')), + ] + + def test_parse_url(self): + self.config(rpc_backend=self.rpc_backend) + + hosts = [] + for host in self.hosts: + hosts.append(oslo_messaging.TransportHost(host.get('hostname'), + host.get('port'), + host.get('username'), + host.get('password'))) + + url = oslo_messaging.TransportURL(self.conf, + self.transport, + self.virtual_host, + hosts, + self.aliases) + + self.assertEqual(self.expected, str(url)) diff --git a/oslo_messaging/tests/test_utils.py b/oslo_messaging/tests/test_utils.py new file mode 100644 index 000000000..9225c94fb --- /dev/null +++ b/oslo_messaging/tests/test_utils.py @@ -0,0 +1,49 @@ + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_messaging import _utils as utils +from oslo_messaging.tests import utils as test_utils + + +class VersionIsCompatibleTestCase(test_utils.BaseTestCase): + def test_version_is_compatible_same(self): + self.assertTrue(utils.version_is_compatible('1.23', '1.23')) + + def test_version_is_compatible_newer_minor(self): + self.assertTrue(utils.version_is_compatible('1.24', '1.23')) + + def test_version_is_compatible_older_minor(self): + self.assertFalse(utils.version_is_compatible('1.22', '1.23')) + + def test_version_is_compatible_major_difference1(self): + self.assertFalse(utils.version_is_compatible('2.23', '1.23')) + + def test_version_is_compatible_major_difference2(self): + self.assertFalse(utils.version_is_compatible('1.23', '2.23')) + + def test_version_is_compatible_newer_rev(self): + self.assertFalse(utils.version_is_compatible('1.23', '1.23.1')) + + def test_version_is_compatible_newer_rev_both(self): + self.assertFalse(utils.version_is_compatible('1.23.1', '1.23.2')) + + def test_version_is_compatible_older_rev_both(self): + self.assertTrue(utils.version_is_compatible('1.23.2', '1.23.1')) + + def test_version_is_compatible_older_rev(self): + self.assertTrue(utils.version_is_compatible('1.24', '1.23.1')) + + def test_version_is_compatible_no_rev_is_zero(self): + self.assertTrue(utils.version_is_compatible('1.23.0', '1.23')) diff --git a/tests/utils.py b/oslo_messaging/tests/utils.py similarity index 97% rename from tests/utils.py rename to oslo_messaging/tests/utils.py index 886532c24..f1b44a60e 100644 --- a/tests/utils.py +++ b/oslo_messaging/tests/utils.py @@ -34,7 +34,7 @@ class BaseTestCase(base.BaseTestCase): def setUp(self, conf=cfg.CONF): super(BaseTestCase, self).setUp() - from oslo.messaging import conffixture + from oslo_messaging import conffixture self.messaging_conf = self.useFixture(conffixture.ConfFixture(conf)) self.messaging_conf.transport_driver = 'fake' self.conf = self.messaging_conf.conf diff --git a/oslo_messaging/transport.py b/oslo_messaging/transport.py new file mode 100644 index 000000000..181cacb4a --- /dev/null +++ b/oslo_messaging/transport.py @@ -0,0 +1,424 @@ + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# Copyright 2013 Red Hat, Inc. +# Copyright (c) 2012 Rackspace Hosting +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__all__ = [ + 'DriverLoadFailure', + 'InvalidTransportURL', + 'Transport', + 'TransportHost', + 'TransportURL', + 'get_transport', + 'set_transport_defaults', +] + +import six +from six.moves.urllib import parse +from stevedore import driver + +from oslo.config import cfg +from oslo_messaging import exceptions + + +_transport_opts = [ + cfg.StrOpt('transport_url', + help='A URL representing the messaging driver to use and its ' + 'full configuration. If not set, we fall back to the ' + 'rpc_backend option and driver specific configuration.'), + cfg.StrOpt('rpc_backend', + default='rabbit', + help='The messaging driver to use, defaults to rabbit. Other ' + 'drivers include qpid and zmq.'), + cfg.StrOpt('control_exchange', + default='openstack', + help='The default exchange under which topics are scoped. May ' + 'be overridden by an exchange name specified in the ' + 'transport_url option.'), +] + + +def set_transport_defaults(control_exchange): + """Set defaults for messaging transport configuration options. + + :param control_exchange: the default exchange under which topics are scoped + :type control_exchange: str + """ + cfg.set_defaults(_transport_opts, + control_exchange=control_exchange) + + +class Transport(object): + + """A messaging transport. + + This is a mostly opaque handle for an underlying messaging transport + driver. + + It has a single 'conf' property which is the cfg.ConfigOpts instance used + to construct the transport object. + """ + + def __init__(self, driver): + self.conf = driver.conf + self._driver = driver + + def _require_driver_features(self, requeue=False): + self._driver.require_features(requeue=requeue) + + def _send(self, target, ctxt, message, wait_for_reply=None, timeout=None, + retry=None): + if not target.topic: + raise exceptions.InvalidTarget('A topic is required to send', + target) + return self._driver.send(target, ctxt, message, + wait_for_reply=wait_for_reply, + timeout=timeout, retry=retry) + + def _send_notification(self, target, ctxt, message, version, retry=None): + if not target.topic: + raise exceptions.InvalidTarget('A topic is required to send', + target) + self._driver.send_notification(target, ctxt, message, version, + retry=retry) + + def _listen(self, target): + if not (target.topic and target.server): + raise exceptions.InvalidTarget('A server\'s target must have ' + 'topic and server names specified', + target) + return self._driver.listen(target) + + def _listen_for_notifications(self, targets_and_priorities, pool): + for target, priority in targets_and_priorities: + if not target.topic: + raise exceptions.InvalidTarget('A target must have ' + 'topic specified', + target) + return self._driver.listen_for_notifications( + targets_and_priorities, pool) + + def cleanup(self): + """Release all resources associated with this transport.""" + self._driver.cleanup() + + +class InvalidTransportURL(exceptions.MessagingException): + """Raised if transport URL is invalid.""" + + def __init__(self, url, msg): + super(InvalidTransportURL, self).__init__(msg) + self.url = url + + +class DriverLoadFailure(exceptions.MessagingException): + """Raised if a transport driver can't be loaded.""" + + def __init__(self, driver, ex): + msg = 'Failed to load transport driver "%s": %s' % (driver, ex) + super(DriverLoadFailure, self).__init__(msg) + self.driver = driver + self.ex = ex + + +def get_transport(conf, url=None, allowed_remote_exmods=None, aliases=None): + """A factory method for Transport objects. + + This method will construct a Transport object from transport configuration + gleaned from the user's configuration and, optionally, a transport URL. + + If a transport URL is supplied as a parameter, any transport configuration + contained in it takes precedence. If no transport URL is supplied, but + there is a transport URL supplied in the user's configuration then that + URL will take the place of the URL parameter. In both cases, any + configuration not supplied in the transport URL may be taken from + individual configuration parameters in the user's configuration. + + An example transport URL might be:: + + rabbit://me:passwd@host:5672/virtual_host + + and can either be passed as a string or a TransportURL object. + + :param conf: the user configuration + :type conf: cfg.ConfigOpts + :param url: a transport URL + :type url: str or TransportURL + :param allowed_remote_exmods: a list of modules which a client using this + transport will deserialize remote exceptions + from + :type allowed_remote_exmods: list + :param aliases: A map of transport alias to transport name + :type aliases: dict + """ + allowed_remote_exmods = allowed_remote_exmods or [] + conf.register_opts(_transport_opts) + + if not isinstance(url, TransportURL): + url = url or conf.transport_url + parsed = TransportURL.parse(conf, url, aliases) + if not parsed.transport: + raise InvalidTransportURL(url, 'No scheme specified in "%s"' % url) + url = parsed + + kwargs = dict(default_exchange=conf.control_exchange, + allowed_remote_exmods=allowed_remote_exmods) + + try: + mgr = driver.DriverManager('oslo.messaging.drivers', + url.transport.split('+')[0], + invoke_on_load=True, + invoke_args=[conf, url], + invoke_kwds=kwargs) + except RuntimeError as ex: + raise DriverLoadFailure(url.transport, ex) + + return Transport(mgr.driver) + + +class TransportHost(object): + + """A host element of a parsed transport URL.""" + + def __init__(self, hostname=None, port=None, username=None, password=None): + self.hostname = hostname + self.port = port + self.username = username + self.password = password + + def __hash__(self): + return hash((self.hostname, self.port, self.username, self.password)) + + def __eq__(self, other): + return vars(self) == vars(other) + + def __ne__(self, other): + return not self == other + + def __repr__(self): + attrs = [] + for a in ['hostname', 'port', 'username', 'password']: + v = getattr(self, a) + if v: + attrs.append((a, repr(v))) + values = ', '.join(['%s=%s' % i for i in attrs]) + return '' + + +class TransportURL(object): + + """A parsed transport URL. + + Transport URLs take the form:: + + transport://user:pass@host1:port[,hostN:portN]/virtual_host + + i.e. the scheme selects the transport driver, you may include multiple + hosts in netloc and the path part is a "virtual host" partition path. + + :param conf: a ConfigOpts instance + :type conf: oslo.config.cfg.ConfigOpts + :param transport: a transport name for example 'rabbit' or 'qpid' + :type transport: str + :param virtual_host: a virtual host path for example '/' + :type virtual_host: str + :param hosts: a list of TransportHost objects + :type hosts: list + :param aliases: A map of transport alias to transport name + :type aliases: dict + """ + + def __init__(self, conf, transport=None, virtual_host=None, hosts=None, + aliases=None): + self.conf = conf + self.conf.register_opts(_transport_opts) + self._transport = transport + self.virtual_host = virtual_host + if hosts is None: + self.hosts = [] + else: + self.hosts = hosts + if aliases is None: + self.aliases = {} + else: + self.aliases = aliases + + @property + def transport(self): + if self._transport is None: + transport = self.conf.rpc_backend + else: + transport = self._transport + return self.aliases.get(transport, transport) + + @transport.setter + def transport(self, value): + self._transport = value + + def __hash__(self): + return hash((tuple(self.hosts), self.transport, self.virtual_host)) + + def __eq__(self, other): + return (self.transport == other.transport and + self.virtual_host == other.virtual_host and + self.hosts == other.hosts) + + def __ne__(self, other): + return not self == other + + def __repr__(self): + attrs = [] + for a in ['transport', 'virtual_host', 'hosts']: + v = getattr(self, a) + if v: + attrs.append((a, repr(v))) + values = ', '.join(['%s=%s' % i for i in attrs]) + return '' + + def __str__(self): + netlocs = [] + + for host in self.hosts: + username = host.username + password = host.password + hostname = host.hostname + port = host.port + + # Starting place for the network location + netloc = '' + + # Build the username and password portion of the transport URL + if username is not None or password is not None: + if username is not None: + netloc += parse.quote(username, '') + if password is not None: + netloc += ':%s' % parse.quote(password, '') + netloc += '@' + + # Build the network location portion of the transport URL + if hostname: + if ':' in hostname: + netloc += '[%s]' % hostname + else: + netloc += hostname + if port is not None: + netloc += ':%d' % port + + netlocs.append(netloc) + + # Assemble the transport URL + url = '%s://%s/' % (self.transport, ','.join(netlocs)) + + if self.virtual_host: + url += parse.quote(self.virtual_host) + + return url + + @classmethod + def parse(cls, conf, url, aliases=None): + """Parse an url. + + Assuming a URL takes the form of:: + + transport://user:pass@host1:port[,hostN:portN]/virtual_host + + then parse the URL and return a TransportURL object. + + Netloc is parsed following the sequence bellow: + + * It is first split by ',' in order to support multiple hosts + * Username and password should be specified for each host, in + case of lack of specification they will be omitted:: + + user:pass@host1:port1,host2:port2 + + [ + {"username": "user", "password": "pass", "host": "host1:port1"}, + {"host": "host2:port2"} + ] + + :param conf: a ConfigOpts instance + :type conf: oslo.config.cfg.ConfigOpts + :param url: The URL to parse + :type url: str + :param aliases: A map of transport alias to transport name + :type aliases: dict + :returns: A TransportURL + """ + if not url: + return cls(conf, aliases=aliases) + + if not isinstance(url, six.string_types): + raise InvalidTransportURL(url, 'Wrong URL type') + + url = parse.urlparse(url) + + # Make sure there's not a query string; that could identify + # requirements we can't comply with (for example ssl), so reject it if + # it's present + if '?' in url.path or url.query: + raise InvalidTransportURL(url.geturl(), + "Cannot comply with query string in " + "transport URL") + + virtual_host = None + if url.path.startswith('/'): + virtual_host = url.path[1:] + + hosts = [] + + username = password = '' + for host in url.netloc.split(','): + if not host: + continue + + hostname = host + username = password = port = None + + if '@' in host: + username, hostname = host.split('@', 1) + if ':' in username: + username, password = username.split(':', 1) + + if not hostname: + hostname = None + elif hostname.startswith('['): + # Find the closing ']' and extract the hostname + host_end = hostname.find(']') + if host_end < 0: + # NOTE(Vek): Identical to what Python 2.7's + # urlparse.urlparse() raises in this case + raise ValueError("Invalid IPv6 URL") + + port_text = hostname[host_end:] + hostname = hostname[1:host_end] + + # Now we need the port; this is compliant with how urlparse + # parses the port data + port = None + if ':' in port_text: + port = int(port_text.split(':', 1)[1]) + elif ':' in hostname: + hostname, port = hostname.split(':', 1) + port = int(port) + + hosts.append(TransportHost(hostname=hostname, + port=port, + username=username, + password=password)) + + return cls(conf, url.scheme, virtual_host, hosts, aliases) diff --git a/setup.cfg b/setup.cfg index a716d98dd..1d58451cc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,39 +20,40 @@ classifier = [files] packages = oslo + oslo_messaging namespace_packages = oslo [entry_points] console_scripts = - oslo-messaging-zmq-receiver = oslo.messaging._cmd.zmq_receiver:main + oslo-messaging-zmq-receiver = oslo_messaging._cmd.zmq_receiver:main oslo.messaging.drivers = - rabbit = oslo.messaging._drivers.impl_rabbit:RabbitDriver - qpid = oslo.messaging._drivers.impl_qpid:QpidDriver - zmq = oslo.messaging._drivers.impl_zmq:ZmqDriver - amqp = oslo.messaging._drivers.protocols.amqp.driver:ProtonDriver + rabbit = oslo_messaging._drivers.impl_rabbit:RabbitDriver + qpid = oslo_messaging._drivers.impl_qpid:QpidDriver + zmq = oslo_messaging._drivers.impl_zmq:ZmqDriver + amqp = oslo_messaging._drivers.protocols.amqp.driver:ProtonDriver # To avoid confusion - kombu = oslo.messaging._drivers.impl_rabbit:RabbitDriver + kombu = oslo_messaging._drivers.impl_rabbit:RabbitDriver # This is just for internal testing - fake = oslo.messaging._drivers.impl_fake:FakeDriver + fake = oslo_messaging._drivers.impl_fake:FakeDriver oslo.messaging.executors = - blocking = oslo.messaging._executors.impl_blocking:BlockingExecutor - eventlet = oslo.messaging._executors.impl_eventlet:EventletExecutor + blocking = oslo_messaging._executors.impl_blocking:BlockingExecutor + eventlet = oslo_messaging._executors.impl_eventlet:EventletExecutor oslo.messaging.notify.drivers = - messagingv2 = oslo.messaging.notify._impl_messaging:MessagingV2Driver - messaging = oslo.messaging.notify._impl_messaging:MessagingDriver - log = oslo.messaging.notify._impl_log:LogDriver - test = oslo.messaging.notify._impl_test:TestDriver - noop = oslo.messaging.notify._impl_noop:NoOpDriver - routing = oslo.messaging.notify._impl_routing:RoutingDriver + messagingv2 = oslo_messaging.notify._impl_messaging:MessagingV2Driver + messaging = oslo_messaging.notify._impl_messaging:MessagingDriver + log = oslo_messaging.notify._impl_log:LogDriver + test = oslo_messaging.notify._impl_test:TestDriver + noop = oslo_messaging.notify._impl_noop:NoOpDriver + routing = oslo_messaging.notify._impl_routing:RoutingDriver oslo.config.opts = - oslo.messaging = oslo.messaging.opts:list_opts + oslo.messaging = oslo_messaging.opts:list_opts [build_sphinx] source-dir = doc/source diff --git a/tests/drivers/test_impl_qpid.py b/tests/drivers/test_impl_qpid.py index 3c22794ae..c8ae40189 100644 --- a/tests/drivers/test_impl_qpid.py +++ b/tests/drivers/test_impl_qpid.py @@ -27,8 +27,8 @@ import testscenarios import testtools from oslo import messaging -from oslo.messaging._drivers import impl_qpid as qpid_driver -from tests import utils as test_utils +from oslo_messaging._drivers import impl_qpid as qpid_driver +from oslo_messaging.tests import utils as test_utils load_tests = testscenarios.load_tests_apply_scenarios diff --git a/tests/drivers/test_impl_rabbit.py b/tests/drivers/test_impl_rabbit.py index 57eeadc7d..1fe005732 100644 --- a/tests/drivers/test_impl_rabbit.py +++ b/tests/drivers/test_impl_rabbit.py @@ -26,11 +26,11 @@ import testscenarios from oslo.config import cfg from oslo import messaging -from oslo.messaging._drivers import amqpdriver -from oslo.messaging._drivers import common as driver_common -from oslo.messaging._drivers import impl_rabbit as rabbit_driver from oslo.serialization import jsonutils -from tests import utils as test_utils +from oslo_messaging._drivers import amqpdriver +from oslo_messaging._drivers import common as driver_common +from oslo_messaging._drivers import impl_rabbit as rabbit_driver +from oslo_messaging.tests import utils as test_utils load_tests = testscenarios.load_tests_apply_scenarios @@ -64,8 +64,8 @@ class TestRabbitDriverLoad(test_utils.BaseTestCase): url='memory:///')) ] - @mock.patch('oslo.messaging._drivers.impl_rabbit.Connection.ensure') - @mock.patch('oslo.messaging._drivers.impl_rabbit.Connection.reset') + @mock.patch('oslo_messaging._drivers.impl_rabbit.Connection.ensure') + @mock.patch('oslo_messaging._drivers.impl_rabbit.Connection.reset') def test_driver_load(self, fake_ensure, fake_reset): self.messaging_conf.transport_driver = self.transport_driver transport = messaging.get_transport(self.conf) @@ -136,8 +136,8 @@ class TestRabbitTransportURL(test_utils.BaseTestCase): super(TestRabbitTransportURL, self).setUp() self.messaging_conf.transport_driver = 'rabbit' - @mock.patch('oslo.messaging._drivers.impl_rabbit.Connection.ensure') - @mock.patch('oslo.messaging._drivers.impl_rabbit.Connection.reset') + @mock.patch('oslo_messaging._drivers.impl_rabbit.Connection.ensure') + @mock.patch('oslo_messaging._drivers.impl_rabbit.Connection.reset') def test_transport_url(self, fake_ensure_connection, fake_reset): transport = messaging.get_transport(self.conf, self.url) self.addCleanup(transport.cleanup) diff --git a/tests/drivers/test_impl_zmq.py b/tests/drivers/test_impl_zmq.py index 3103f42aa..0f63d891b 100644 --- a/tests/drivers/test_impl_zmq.py +++ b/tests/drivers/test_impl_zmq.py @@ -22,7 +22,7 @@ import testtools from oslo import messaging from oslo.utils import importutils -from tests import utils as test_utils +from oslo_messaging.tests import utils as test_utils # NOTE(jamespage) the zmq driver implementation is currently tied # to eventlet so we have to monkey_patch to support testing diff --git a/tests/drivers/test_matchmaker.py b/tests/drivers/test_matchmaker.py index 0de2e6d2f..767414509 100644 --- a/tests/drivers/test_matchmaker.py +++ b/tests/drivers/test_matchmaker.py @@ -15,7 +15,7 @@ import testtools from oslo.utils import importutils -from tests import utils as test_utils +from oslo_messaging.tests import utils as test_utils # NOTE(jamespage) matchmaker tied directly to eventlet # which is not yet py3 compatible - skip if import fails diff --git a/tests/drivers/test_matchmaker_redis.py b/tests/drivers/test_matchmaker_redis.py index 1c20dabde..95bde9e8a 100644 --- a/tests/drivers/test_matchmaker_redis.py +++ b/tests/drivers/test_matchmaker_redis.py @@ -15,7 +15,7 @@ import testtools from oslo.utils import importutils -from tests import utils as test_utils +from oslo_messaging.tests import utils as test_utils redis = importutils.try_import('redis') matchmaker_redis = ( diff --git a/tests/drivers/test_matchmaker_ring.py b/tests/drivers/test_matchmaker_ring.py index c6d81e6eb..c3bc52493 100644 --- a/tests/drivers/test_matchmaker_ring.py +++ b/tests/drivers/test_matchmaker_ring.py @@ -15,7 +15,7 @@ import testtools from oslo.utils import importutils -from tests import utils as test_utils +from oslo_messaging.tests import utils as test_utils # NOTE(jamespage) matchmaker tied directly to eventlet # which is not yet py3 compatible - skip if import fails diff --git a/tests/drivers/test_pool.py b/tests/drivers/test_pool.py index e068f84a9..a0b6ab47c 100644 --- a/tests/drivers/test_pool.py +++ b/tests/drivers/test_pool.py @@ -18,8 +18,8 @@ import uuid import testscenarios -from oslo.messaging._drivers import pool -from tests import utils as test_utils +from oslo_messaging._drivers import pool +from oslo_messaging.tests import utils as test_utils load_tests = testscenarios.load_tests_apply_scenarios diff --git a/tests/functional/utils.py b/tests/functional/utils.py index 8c5751895..fc0c78025 100644 --- a/tests/functional/utils.py +++ b/tests/functional/utils.py @@ -22,7 +22,7 @@ from six import moves from oslo.config import cfg from oslo import messaging from oslo.messaging.notify import notifier -from tests import utils as test_utils +from oslo_messaging.tests import utils as test_utils class TestServerEndpoint(object): diff --git a/tests/notify/test_dispatcher.py b/tests/notify/test_dispatcher.py index 791794887..f2e32ac48 100644 --- a/tests/notify/test_dispatcher.py +++ b/tests/notify/test_dispatcher.py @@ -21,7 +21,7 @@ import testscenarios from oslo import messaging from oslo.messaging.notify import dispatcher as notify_dispatcher from oslo.utils import timeutils -from tests import utils as test_utils +from oslo_messaging.tests import utils as test_utils load_tests = testscenarios.load_tests_apply_scenarios @@ -140,7 +140,7 @@ class TestDispatcherScenario(test_utils.BaseTestCase): class TestDispatcher(test_utils.BaseTestCase): - @mock.patch('oslo.messaging.notify.dispatcher.LOG') + @mock.patch('oslo_messaging.notify.dispatcher.LOG') def test_dispatcher_unknown_prio(self, mylog): msg = notification_msg.copy() msg['priority'] = 'what???' diff --git a/tests/notify/test_listener.py b/tests/notify/test_listener.py index 72e142e7e..9317eaeff 100644 --- a/tests/notify/test_listener.py +++ b/tests/notify/test_listener.py @@ -21,7 +21,7 @@ import testscenarios from oslo.config import cfg from oslo import messaging from oslo.messaging.notify import dispatcher -from tests import utils as test_utils +from oslo_messaging.tests import utils as test_utils load_tests = testscenarios.load_tests_apply_scenarios @@ -63,6 +63,8 @@ class ListenerSetupMixin(object): self.stop() def wait_for(self, expect_messages): + print('expecting %d messages have %d' % + (expect_messages, self._received_msgs)) while expect_messages != self._received_msgs: yield diff --git a/tests/notify/test_log_handler.py b/tests/notify/test_log_handler.py index 4e676f210..3b89af01a 100644 --- a/tests/notify/test_log_handler.py +++ b/tests/notify/test_log_handler.py @@ -16,8 +16,8 @@ import mock from oslo import messaging from oslo.messaging.notify import log_handler -from tests.notify import test_notifier -from tests import utils as test_utils +from oslo_messaging.tests.notify import test_notifier +from oslo_messaging.tests import utils as test_utils class PublishErrorsHandlerTestCase(test_utils.BaseTestCase): @@ -46,7 +46,7 @@ class PublishErrorsHandlerTestCase(test_utils.BaseTestCase): self.publisherrorshandler.emit(logrecord) self.assertTrue(self.stub_flg) - @mock.patch.object(messaging.notify.notifier.Notifier, '_notify') + @mock.patch('oslo_messaging.notify.notifier.Notifier._notify') def test_emit_notification(self, mock_notify): logrecord = logging.LogRecord(name='name', level='ERROR', pathname='/tmp', lineno=1, msg='Message', diff --git a/tests/notify/test_logger.py b/tests/notify/test_logger.py index 668e96312..1f770bded 100644 --- a/tests/notify/test_logger.py +++ b/tests/notify/test_logger.py @@ -24,8 +24,9 @@ import testtools from oslo import messaging from oslo.utils import timeutils -from tests.notify import test_notifier -from tests import utils as test_utils +import oslo_messaging +from oslo_messaging.tests.notify import test_notifier +from oslo_messaging.tests import utils as test_utils load_tests = testscenarios.load_tests_apply_scenarios @@ -49,7 +50,7 @@ class TestLogNotifier(test_utils.BaseTestCase): def setUp(self): super(TestLogNotifier, self).setUp() - self.addCleanup(messaging.notify._impl_test.reset) + self.addCleanup(oslo_messaging.notify._impl_test.reset) self.config(notification_driver=['test']) # NOTE(jamespage) disable thread information logging for testing # as this causes test failures when zmq tests monkey_patch via @@ -58,7 +59,7 @@ class TestLogNotifier(test_utils.BaseTestCase): @mock.patch('oslo.utils.timeutils.utcnow') def test_logger(self, mock_utcnow): - with mock.patch('oslo.messaging.transport.get_transport', + with mock.patch('oslo_messaging.transport.get_transport', return_value=test_notifier._FakeTransport(self.conf)): self.logger = messaging.LoggingNotificationHandler('test://') @@ -76,7 +77,7 @@ class TestLogNotifier(test_utils.BaseTestCase): self.logger.emit(record) - n = messaging.notify._impl_test.NOTIFICATIONS[0][1] + n = oslo_messaging.notify._impl_test.NOTIFICATIONS[0][1] self.assertEqual(getattr(self, 'queue', self.priority.upper()), n['priority']) self.assertEqual('logrecord', n['event_type']) @@ -101,7 +102,7 @@ class TestLogNotifier(test_utils.BaseTestCase): "Need logging.config.dictConfig (Python >= 2.7)") @mock.patch('oslo.utils.timeutils.utcnow') def test_logging_conf(self, mock_utcnow): - with mock.patch('oslo.messaging.transport.get_transport', + with mock.patch('oslo_messaging.transport.get_transport', return_value=test_notifier._FakeTransport(self.conf)): logging.config.dictConfig({ 'version': 1, @@ -128,7 +129,7 @@ class TestLogNotifier(test_utils.BaseTestCase): lineno = sys._getframe().f_lineno + 1 logger.log(levelno, 'foobar') - n = messaging.notify._impl_test.NOTIFICATIONS[0][1] + n = oslo_messaging.notify._impl_test.NOTIFICATIONS[0][1] self.assertEqual(getattr(self, 'queue', self.priority.upper()), n['priority']) self.assertEqual('logrecord', n['event_type']) diff --git a/tests/notify/test_middleware.py b/tests/notify/test_middleware.py index 0eb3cac2e..8bdd1676e 100644 --- a/tests/notify/test_middleware.py +++ b/tests/notify/test_middleware.py @@ -19,7 +19,7 @@ import mock import webob from oslo.messaging.notify import middleware -from tests import utils +from oslo_messaging.tests import utils class FakeApp(object): diff --git a/tests/notify/test_notifier.py b/tests/notify/test_notifier.py index 40c888d41..e7306a714 100644 --- a/tests/notify/test_notifier.py +++ b/tests/notify/test_notifier.py @@ -26,14 +26,14 @@ import testscenarios import yaml from oslo import messaging -from oslo.messaging.notify import _impl_log -from oslo.messaging.notify import _impl_messaging -from oslo.messaging.notify import _impl_test -from oslo.messaging.notify import notifier as msg_notifier from oslo.messaging import serializer as msg_serializer from oslo.serialization import jsonutils from oslo.utils import timeutils -from tests import utils as test_utils +from oslo_messaging.notify import _impl_log +from oslo_messaging.notify import _impl_messaging +from oslo_messaging.notify import _impl_test +from oslo_messaging.notify import notifier as msg_notifier +from oslo_messaging.tests import utils as test_utils load_tests = testscenarios.load_tests_apply_scenarios @@ -358,7 +358,7 @@ class TestRoutingNotifier(test_utils.BaseTestCase): config_file): with mock.patch('stevedore.dispatch.DispatchExtensionManager', return_value=self._empty_extension_manager()): - with mock.patch('oslo.messaging.notify.' + with mock.patch('oslo_messaging.notify.' '_impl_routing.LOG') as mylog: self.router._load_notifiers() self.assertFalse(mylog.debug.called) diff --git a/tests/rpc/test_client.py b/tests/rpc/test_client.py index e441e916e..5f138925e 100644 --- a/tests/rpc/test_client.py +++ b/tests/rpc/test_client.py @@ -19,7 +19,7 @@ from oslo.config import cfg from oslo import messaging from oslo.messaging import exceptions from oslo.messaging import serializer as msg_serializer -from tests import utils as test_utils +from oslo_messaging.tests import utils as test_utils load_tests = testscenarios.load_tests_apply_scenarios diff --git a/tests/rpc/test_dispatcher.py b/tests/rpc/test_dispatcher.py index 2f1fb4d5f..c3925e6b3 100644 --- a/tests/rpc/test_dispatcher.py +++ b/tests/rpc/test_dispatcher.py @@ -18,7 +18,7 @@ import testscenarios from oslo import messaging from oslo.messaging import serializer as msg_serializer -from tests import utils as test_utils +from oslo_messaging.tests import utils as test_utils load_tests = testscenarios.load_tests_apply_scenarios diff --git a/tests/rpc/test_server.py b/tests/rpc/test_server.py index b4bdcccec..d5f879c49 100644 --- a/tests/rpc/test_server.py +++ b/tests/rpc/test_server.py @@ -20,7 +20,7 @@ import testscenarios from oslo.config import cfg from oslo import messaging -from tests import utils as test_utils +from oslo_messaging.tests import utils as test_utils load_tests = testscenarios.load_tests_apply_scenarios diff --git a/tests/test_amqp_driver.py b/tests/test_amqp_driver.py index 64bbd3003..755b3da7b 100644 --- a/tests/test_amqp_driver.py +++ b/tests/test_amqp_driver.py @@ -25,7 +25,7 @@ import testtools from oslo import messaging from oslo.utils import importutils -from tests import utils as test_utils +from oslo_messaging.tests import utils as test_utils # TODO(kgiusti) Conditionally run these tests only if the necessary # dependencies are installed. This should be removed once the proton libraries diff --git a/tests/test_exception_serialization.py b/tests/test_exception_serialization.py index 7c3d2a05f..74d808f05 100644 --- a/tests/test_exception_serialization.py +++ b/tests/test_exception_serialization.py @@ -19,9 +19,9 @@ import six import testscenarios from oslo import messaging -from oslo.messaging._drivers import common as exceptions from oslo.serialization import jsonutils -from tests import utils as test_utils +from oslo_messaging._drivers import common as exceptions +from oslo_messaging.tests import utils as test_utils load_tests = testscenarios.load_tests_apply_scenarios diff --git a/tests/test_expected_exceptions.py b/tests/test_expected_exceptions.py index fe29ec063..702f3a2b3 100644 --- a/tests/test_expected_exceptions.py +++ b/tests/test_expected_exceptions.py @@ -15,7 +15,7 @@ # under the License. from oslo import messaging -from tests import utils as test_utils +from oslo_messaging.tests import utils as test_utils class TestExpectedExceptions(test_utils.BaseTestCase): diff --git a/tests/test_target.py b/tests/test_target.py index bdf857034..68f98f4d7 100644 --- a/tests/test_target.py +++ b/tests/test_target.py @@ -16,7 +16,7 @@ import testscenarios from oslo import messaging -from tests import utils as test_utils +from oslo_messaging.tests import utils as test_utils load_tests = testscenarios.load_tests_apply_scenarios diff --git a/tests/test_transport.py b/tests/test_transport.py index 6e700e88a..5a9a73c83 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -22,7 +22,8 @@ import testscenarios from oslo.config import cfg from oslo import messaging from oslo.messaging import transport -from tests import utils as test_utils +from oslo_messaging.tests import utils as test_utils +from oslo_messaging import transport as private_transport load_tests = testscenarios.load_tests_apply_scenarios @@ -241,7 +242,7 @@ class TestSetDefaults(test_utils.BaseTestCase): def setUp(self): super(TestSetDefaults, self).setUp(conf=cfg.ConfigOpts()) self.useFixture(_SetDefaultsFixture(messaging.set_transport_defaults, - transport._transport_opts, + private_transport._transport_opts, 'control_exchange')) def test_set_default_control_exchange(self): diff --git a/tests/test_urls.py b/tests/test_urls.py index c35dad34c..956274201 100644 --- a/tests/test_urls.py +++ b/tests/test_urls.py @@ -16,7 +16,7 @@ import testscenarios from oslo import messaging -from tests import utils as test_utils +from oslo_messaging.tests import utils as test_utils load_tests = testscenarios.load_tests_apply_scenarios diff --git a/tests/test_utils.py b/tests/test_utils.py index 38ee1d0ca..d57e32ed2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -13,9 +13,9 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo.messaging._drivers import common -from oslo.messaging import _utils as utils -from tests import utils as test_utils +from oslo_messaging._drivers import common +from oslo_messaging import _utils as utils +from oslo_messaging.tests import utils as test_utils class VersionIsCompatibleTestCase(test_utils.BaseTestCase): diff --git a/tests/test_warning.py b/tests/test_warning.py new file mode 100644 index 000000000..136f487c4 --- /dev/null +++ b/tests/test_warning.py @@ -0,0 +1,61 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import imp +import os +import warnings + +import mock +from oslotest import base as test_base +import six + + +class DeprecationWarningTest(test_base.BaseTestCase): + + @mock.patch('warnings.warn') + def test_warning(self, mock_warn): + import oslo.messaging + imp.reload(oslo.messaging) + self.assertTrue(mock_warn.called) + args = mock_warn.call_args + self.assertIn('oslo_messaging', args[0][0]) + self.assertIn('deprecated', args[0][0]) + self.assertTrue(issubclass(args[0][1], DeprecationWarning)) + + def test_real_warning(self): + with warnings.catch_warnings(record=True) as warning_msgs: + warnings.resetwarnings() + warnings.simplefilter('always', DeprecationWarning) + import oslo.messaging + + # Use a separate function to get the stack level correct + # so we know the message points back to this file. This + # corresponds to an import or reload, which isn't working + # inside the test under Python 3.3. That may be due to a + # difference in the import implementation not triggering + # warnings properly when the module is reloaded, or + # because the warnings module is mostly implemented in C + # and something isn't cleanly resetting the global state + # used to track whether a warning needs to be + # emitted. Whatever the cause, we definitely see the + # warnings.warn() being invoked on a reload (see the test + # above) and warnings are reported on the console when we + # run the tests. A simpler test script run outside of + # testr does correctly report the warnings. + def foo(): + oslo.messaging.deprecated() + + foo() + self.assertEqual(1, len(warning_msgs)) + msg = warning_msgs[0] + self.assertIn('oslo_messaging', six.text_type(msg.message)) + self.assertEqual('test_warning.py', os.path.basename(msg.filename)) diff --git a/tox.ini b/tox.ini index dd0f1da45..428b051bf 100644 --- a/tox.ini +++ b/tox.ini @@ -54,5 +54,5 @@ exclude = .tox,dist,doc,*.egg,build,__init__.py [hacking] import_exceptions = - oslo.messaging._i18n + oslo_messaging._i18n six.moves