From 32d088b2c1b9b012fadc7e2870abe91f8f792298 Mon Sep 17 00:00:00 2001 From: Tovin Seven Date: Tue, 17 Jan 2017 16:44:37 +0700 Subject: [PATCH] Integrate OSProfiler in Magnum * Add osprofiler wsgi middleware. This middleware is used for 2 things: 1) It checks that person who wants to trace is trusted and knows secret HMAC key. 2) It starts tracing in case of proper trace headers and adds first wsgi trace point, with info about HTTP request * Add initialization of osprofiler at start of service Currently that includes oslo.messaging notifer instance creation to send Ceilometer backend notifications. * Traces HTTP/RPC/DB API calls Demo: https://hieulq.github.io/cluster-create-false-new-html.html Co-Authored-By: Hieu LE Implements: blueprint osprofiler-support-in-magnum Change-Id: I7d68995aab81d365433950aada078ef1fcd5469b --- etc/magnum/api-paste.ini | 5 +- magnum/cmd/__init__.py | 20 ++ magnum/cmd/api.py | 4 + magnum/common/config.py | 14 +- magnum/common/context.py | 10 + magnum/common/profiler.py | 86 ++++++ magnum/common/rpc.py | 44 ++- magnum/common/rpc_service.py | 30 ++- magnum/conductor/api.py | 3 + magnum/conductor/handlers/ca_conductor.py | 2 + .../conductor/handlers/cluster_conductor.py | 2 + .../conductor/handlers/conductor_listener.py | 3 + magnum/conductor/handlers/indirection_api.py | 2 + magnum/conductor/monitors.py | 2 + magnum/conf/__init__.py | 2 + magnum/conf/profiler.py | 27 ++ magnum/db/api.py | 3 + magnum/db/sqlalchemy/api.py | 8 + magnum/service/periodic.py | 2 + magnum/tests/unit/common/test_profiler.py | 75 ++++++ magnum/tests/unit/common/test_rpc.py | 252 ++++++++++++++++++ setup.cfg | 4 + test-requirements.txt | 1 + 23 files changed, 577 insertions(+), 24 deletions(-) create mode 100644 magnum/common/profiler.py create mode 100644 magnum/conf/profiler.py create mode 100644 magnum/tests/unit/common/test_profiler.py create mode 100644 magnum/tests/unit/common/test_rpc.py diff --git a/etc/magnum/api-paste.ini b/etc/magnum/api-paste.ini index 31ba068f80..d1f56fb7f4 100644 --- a/etc/magnum/api-paste.ini +++ b/etc/magnum/api-paste.ini @@ -1,5 +1,5 @@ [pipeline:main] -pipeline = cors healthcheck http_proxy_to_wsgi request_id authtoken api_v1 +pipeline = cors healthcheck http_proxy_to_wsgi request_id osprofiler authtoken api_v1 [app:api_v1] paste.app_factory = magnum.api.app:app_factory @@ -8,6 +8,9 @@ paste.app_factory = magnum.api.app:app_factory acl_public_routes = /, /v1 paste.filter_factory = magnum.api.middleware.auth_token:AuthTokenMiddleware.factory +[filter:osprofiler] +paste.filter_factory = magnum.common.profiler:WsgiMiddleware.factory + [filter:request_id] paste.filter_factory = oslo_middleware:RequestId.factory diff --git a/magnum/cmd/__init__.py b/magnum/cmd/__init__.py index e69de29bb2..277b2af39d 100644 --- a/magnum/cmd/__init__.py +++ b/magnum/cmd/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2017 Fujitsu Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# NOTE(hieulq): we monkey patch all eventlet services for easier tracking/debug + +import eventlet + +eventlet.monkey_patch() diff --git a/magnum/cmd/api.py b/magnum/cmd/api.py index 41e54d1211..394f67a5fe 100644 --- a/magnum/cmd/api.py +++ b/magnum/cmd/api.py @@ -22,6 +22,7 @@ from oslo_reports import guru_meditation_report as gmr from werkzeug import serving from magnum.api import app as api_app +from magnum.common import profiler from magnum.common import service import magnum.conf from magnum.i18n import _ @@ -62,6 +63,9 @@ def main(): app = api_app.load_app() + # Setup OSprofiler for WSGI service + profiler.setup('magnum-api', CONF.host) + # SSL configuration use_ssl = CONF.api.enabled_ssl diff --git a/magnum/common/config.py b/magnum/common/config.py index f9f54e6e79..c84645332d 100644 --- a/magnum/common/config.py +++ b/magnum/common/config.py @@ -15,20 +15,22 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo_config import cfg from oslo_middleware import cors from magnum.common import rpc +import magnum.conf from magnum import version +CONF = magnum.conf.CONF + def parse_args(argv, default_config_files=None): rpc.set_defaults(control_exchange='magnum') - cfg.CONF(argv[1:], - project='magnum', - version=version.version_info.release_string(), - default_config_files=default_config_files) - rpc.init(cfg.CONF) + CONF(argv[1:], + project='magnum', + version=version.version_info.release_string(), + default_config_files=default_config_files) + rpc.init(CONF) def set_config_defaults(): diff --git a/magnum/common/context.py b/magnum/common/context.py index a663a06c36..7a4e4edea9 100644 --- a/magnum/common/context.py +++ b/magnum/common/context.py @@ -142,3 +142,13 @@ def set_ctx(new_ctx): if new_ctx: setattr(_CTX_STORE, _CTX_KEY, new_ctx) setattr(context._request_store, 'context', new_ctx) + + +def get_admin_context(read_deleted="no"): + # NOTE(tovin07): This method should only be used when an admin context is + # necessary for the entirety of the context lifetime. + return RequestContext(user_id=None, + project_id=None, + is_admin=True, + read_deleted=read_deleted, + overwrite=False) diff --git a/magnum/common/profiler.py b/magnum/common/profiler.py new file mode 100644 index 0000000000..a529df48e2 --- /dev/null +++ b/magnum/common/profiler.py @@ -0,0 +1,86 @@ +# Copyright 2017 Fujitsu Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +### +# This code is taken from nova. Goal is minimal modification. +### + +from oslo_log import log as logging +from oslo_utils import importutils +import webob.dec + +from magnum.common import context +import magnum.conf +from magnum.i18n import _LI + +profiler = importutils.try_import("osprofiler.profiler") +profiler_initializer = importutils.try_import("osprofiler.initializer") +profiler_web = importutils.try_import("osprofiler.web") + + +CONF = magnum.conf.CONF + +LOG = logging.getLogger(__name__) + + +class WsgiMiddleware(object): + + def __init__(self, application, **kwargs): + self.application = application + + @classmethod + def factory(cls, global_conf, **local_conf): + if profiler_web: + return profiler_web.WsgiMiddleware.factory(global_conf, + **local_conf) + + def filter_(app): + return cls(app, **local_conf) + + return filter_ + + @webob.dec.wsgify + def __call__(self, request): + return request.get_response(self.application) + + +def setup(binary, host): + if CONF.profiler.enabled: + profiler_initializer.init_from_conf( + conf=CONF, + context=context.get_admin_context().to_dict(), + project="magnum", + service=binary, + host=host) + LOG.info(_LI("OSprofiler is enabled.")) + + +def trace_cls(name, **kwargs): + """Wrap the OSprofiler trace_cls. + + Wrap the OSprofiler trace_cls decorator so that it will not try to + patch the class unless OSprofiler is present. + + :param name: The name of action. For example, wsgi, rpc, db, ... + :param kwargs: Any other keyword args used by profiler.trace_cls + """ + + def decorator(cls): + if profiler and 'profiler' in CONF: + trace_decorator = profiler.trace_cls(name, kwargs) + return trace_decorator(cls) + return cls + + return decorator diff --git a/magnum/common/rpc.py b/magnum/common/rpc.py index 22b2bc9ec8..a864160cbc 100644 --- a/magnum/common/rpc.py +++ b/magnum/common/rpc.py @@ -32,11 +32,13 @@ import socket import oslo_messaging as messaging from oslo_serialization import jsonutils +from oslo_utils import importutils from magnum.common import context as magnum_context from magnum.common import exception import magnum.conf +profiler = importutils.try_import("osprofiler.profiler") CONF = magnum.conf.CONF TRANSPORT = None @@ -121,22 +123,56 @@ class RequestContextSerializer(messaging.Serializer): return magnum_context.RequestContext.from_dict(context) +class ProfilerRequestContextSerializer(RequestContextSerializer): + def serialize_context(self, context): + _context = super(ProfilerRequestContextSerializer, + self).serialize_context(context) + + prof = profiler.get() + if prof: + trace_info = { + "hmac_key": prof.hmac_key, + "base_id": prof.get_base_id(), + "parent_id": prof.get_id() + } + _context.update({"trace_info": trace_info}) + + return _context + + def deserialize_context(self, context): + trace_info = context.pop("trace_info", None) + if trace_info: + profiler.init(**trace_info) + + return super(ProfilerRequestContextSerializer, + self).deserialize_context(context) + + def get_transport_url(url_str=None): return messaging.TransportURL.parse(CONF, url_str, TRANSPORT_ALIASES) -def get_client(target, version_cap=None, serializer=None): +def get_client(target, version_cap=None, serializer=None, timeout=None): assert TRANSPORT is not None - serializer = RequestContextSerializer(serializer) + if profiler: + serializer = ProfilerRequestContextSerializer(serializer) + else: + serializer = RequestContextSerializer(serializer) + return messaging.RPCClient(TRANSPORT, target, version_cap=version_cap, - serializer=serializer) + serializer=serializer, + timeout=timeout) def get_server(target, endpoints, serializer=None): assert TRANSPORT is not None - serializer = RequestContextSerializer(serializer) + if profiler: + serializer = ProfilerRequestContextSerializer(serializer) + else: + serializer = RequestContextSerializer(serializer) + return messaging.get_rpc_server(TRANSPORT, target, endpoints, diff --git a/magnum/common/rpc_service.py b/magnum/common/rpc_service.py index 7e03c59428..26d4f3394f 100644 --- a/magnum/common/rpc_service.py +++ b/magnum/common/rpc_service.py @@ -14,24 +14,17 @@ """Common RPC service and API tools for Magnum.""" -import eventlet import oslo_messaging as messaging from oslo_service import service +from oslo_utils import importutils +from magnum.common import profiler from magnum.common import rpc import magnum.conf from magnum.objects import base as objects_base from magnum.service import periodic from magnum.servicegroup import magnum_service_periodic as servicegroup - -# NOTE(paulczar): -# Ubuntu 14.04 forces librabbitmq when kombu is used -# Unfortunately it forces a version that has a crash -# bug. Calling eventlet.monkey_patch() tells kombu -# to use libamqp instead. -eventlet.monkey_patch() - # NOTE(asalkeld): # The magnum.openstack.common.rpc entries are for compatibility # with devstack rpc_backend configuration values. @@ -41,15 +34,26 @@ TRANSPORT_ALIASES = { 'magnum.openstack.common.rpc.impl_zmq': 'zmq', } +osprofiler = importutils.try_import("osprofiler.profiler") + CONF = magnum.conf.CONF +def _init_serializer(): + serializer = rpc.RequestContextSerializer( + objects_base.MagnumObjectSerializer()) + if osprofiler: + serializer = rpc.ProfilerRequestContextSerializer(serializer) + else: + serializer = rpc.RequestContextSerializer(serializer) + return serializer + + class Service(service.Service): def __init__(self, topic, server, handlers, binary): super(Service, self).__init__() - serializer = rpc.RequestContextSerializer( - objects_base.MagnumObjectSerializer()) + serializer = _init_serializer() transport = messaging.get_transport(CONF, aliases=TRANSPORT_ALIASES) # TODO(asalkeld) add support for version='x.y' @@ -57,6 +61,7 @@ class Service(service.Service): self._server = messaging.get_rpc_server(transport, target, handlers, serializer=serializer) self.binary = binary + profiler.setup(binary, CONF.host) def start(self): # NOTE(suro-patz): The parent class has created a threadgroup, already @@ -80,8 +85,7 @@ class Service(service.Service): class API(object): def __init__(self, transport=None, context=None, topic=None, server=None, timeout=None): - serializer = rpc.RequestContextSerializer( - objects_base.MagnumObjectSerializer()) + serializer = _init_serializer() if transport is None: exmods = rpc.get_allowed_exmods() transport = messaging.get_transport(CONF, diff --git a/magnum/conductor/api.py b/magnum/conductor/api.py index 4d70616140..0ce71cf763 100644 --- a/magnum/conductor/api.py +++ b/magnum/conductor/api.py @@ -12,6 +12,7 @@ """API for interfacing with Magnum Backend.""" +from magnum.common import profiler from magnum.common import rpc_service import magnum.conf @@ -22,6 +23,7 @@ CONF = magnum.conf.CONF # API to trigger operations on the conductors +@profiler.trace_cls("rpc") class API(rpc_service.API): def __init__(self, transport=None, context=None, topic=None): super(API, self).__init__(transport, context, @@ -78,6 +80,7 @@ class API(rpc_service.API): target_version=target_version) +@profiler.trace_cls("rpc") class ListenerAPI(rpc_service.API): def __init__(self, context=None, topic=None, server=None, timeout=None): super(ListenerAPI, self).__init__(context=context, topic=topic, diff --git a/magnum/conductor/handlers/ca_conductor.py b/magnum/conductor/handlers/ca_conductor.py index da96979cb0..aebbb1037d 100644 --- a/magnum/conductor/handlers/ca_conductor.py +++ b/magnum/conductor/handlers/ca_conductor.py @@ -15,11 +15,13 @@ from oslo_log import log as logging +from magnum.common import profiler from magnum.conductor.handlers.common import cert_manager from magnum import objects LOG = logging.getLogger(__name__) +@profiler.trace_cls("rpc") class Handler(object): """Magnum CA RPC handler. diff --git a/magnum/conductor/handlers/cluster_conductor.py b/magnum/conductor/handlers/cluster_conductor.py index c531e78ea6..2e1ec9d6c8 100644 --- a/magnum/conductor/handlers/cluster_conductor.py +++ b/magnum/conductor/handlers/cluster_conductor.py @@ -19,6 +19,7 @@ import six from magnum.common import clients from magnum.common import exception +from magnum.common import profiler from magnum.conductor.handlers.common import cert_manager from magnum.conductor.handlers.common import trust_manager from magnum.conductor import scale_manager @@ -35,6 +36,7 @@ CONF = magnum.conf.CONF LOG = logging.getLogger(__name__) +@profiler.trace_cls("rpc") class Handler(object): def __init__(self): diff --git a/magnum/conductor/handlers/conductor_listener.py b/magnum/conductor/handlers/conductor_listener.py index d343caa0a2..fd7710ac81 100644 --- a/magnum/conductor/handlers/conductor_listener.py +++ b/magnum/conductor/handlers/conductor_listener.py @@ -10,7 +10,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from magnum.common import profiler + +@profiler.trace_cls("rpc") class Handler(object): '''Listen on an AMQP queue named for the conductor. diff --git a/magnum/conductor/handlers/indirection_api.py b/magnum/conductor/handlers/indirection_api.py index 6a76ed07e7..1671c984a0 100644 --- a/magnum/conductor/handlers/indirection_api.py +++ b/magnum/conductor/handlers/indirection_api.py @@ -12,9 +12,11 @@ import oslo_messaging as messaging +from magnum.common import profiler from magnum.objects import base +@profiler.trace_cls("rpc") class Handler(object): "Indirection API callbacks" diff --git a/magnum/conductor/monitors.py b/magnum/conductor/monitors.py index f39c39b43a..53602048a0 100644 --- a/magnum/conductor/monitors.py +++ b/magnum/conductor/monitors.py @@ -18,6 +18,7 @@ import abc from oslo_log import log import six +from magnum.common import profiler import magnum.conf from magnum.drivers.common.driver import Driver @@ -27,6 +28,7 @@ LOG = log.getLogger(__name__) CONF = magnum.conf.CONF +@profiler.trace_cls("rpc") @six.add_metaclass(abc.ABCMeta) class MonitorBase(object): diff --git a/magnum/conf/__init__.py b/magnum/conf/__init__.py index 017fa27221..35b4cb07c3 100644 --- a/magnum/conf/__init__.py +++ b/magnum/conf/__init__.py @@ -33,6 +33,7 @@ from magnum.conf import magnum_client from magnum.conf import neutron from magnum.conf import nova from magnum.conf import paths +from magnum.conf import profiler from magnum.conf import quota from magnum.conf import rpc from magnum.conf import services @@ -66,3 +67,4 @@ services.register_opts(CONF) trust.register_opts(CONF) utils.register_opts(CONF) x509.register_opts(CONF) +profiler.register_opts(CONF) diff --git a/magnum/conf/profiler.py b/magnum/conf/profiler.py new file mode 100644 index 0000000000..d0f68c0fb6 --- /dev/null +++ b/magnum/conf/profiler.py @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_utils import importutils + + +profiler_opts = importutils.try_import('osprofiler.opts') + + +def register_opts(conf): + if profiler_opts: + profiler_opts.set_defaults(conf) + + +def list_opts(): + return { + profiler_opts._profiler_opt_group: profiler_opts._PROFILER_OPTS + } diff --git a/magnum/db/api.py b/magnum/db/api.py index 58f80a43ff..aed7c96331 100644 --- a/magnum/db/api.py +++ b/magnum/db/api.py @@ -21,6 +21,8 @@ from oslo_config import cfg from oslo_db import api as db_api import six +from magnum.common import profiler + _BACKEND_MAPPING = {'sqlalchemy': 'magnum.db.sqlalchemy.api'} IMPL = db_api.DBAPI.from_config(cfg.CONF, backend_mapping=_BACKEND_MAPPING, @@ -32,6 +34,7 @@ def get_instance(): return IMPL +@profiler.trace_cls("db") @six.add_metaclass(abc.ABCMeta) class Connection(object): """Base class for storage system connections.""" diff --git a/magnum/db/sqlalchemy/api.py b/magnum/db/sqlalchemy/api.py index 7f75a91da6..38b4e441f0 100644 --- a/magnum/db/sqlalchemy/api.py +++ b/magnum/db/sqlalchemy/api.py @@ -17,9 +17,11 @@ from oslo_db import exception as db_exc from oslo_db.sqlalchemy import session as db_session from oslo_db.sqlalchemy import utils as db_utils +from oslo_utils import importutils from oslo_utils import strutils from oslo_utils import timeutils from oslo_utils import uuidutils +import sqlalchemy as sa from sqlalchemy.orm.exc import MultipleResultsFound from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.sql import func @@ -30,6 +32,8 @@ from magnum.db import api from magnum.db.sqlalchemy import models from magnum.i18n import _ +profiler_sqlalchemy = importutils.try_import('osprofiler.sqlalchemy') + CONF = magnum.conf.CONF @@ -40,6 +44,10 @@ def _create_facade_lazily(): global _FACADE if _FACADE is None: _FACADE = db_session.EngineFacade.from_config(CONF) + if profiler_sqlalchemy: + if CONF.profiler.enabled and CONF.profiler.trace_sqlalchemy: + profiler_sqlalchemy.add_tracing(sa, _FACADE.get_engine(), "db") + return _FACADE diff --git a/magnum/service/periodic.py b/magnum/service/periodic.py index 46d64a7cb4..01f05563f8 100644 --- a/magnum/service/periodic.py +++ b/magnum/service/periodic.py @@ -22,6 +22,7 @@ from oslo_service import periodic_task from pycadf import cadftaxonomy as taxonomy from magnum.common import context +from magnum.common import profiler from magnum.common import rpc from magnum.conductor import monitors from magnum.conductor import utils as conductor_utils @@ -87,6 +88,7 @@ class ClusterUpdateJob(object): raise loopingcall.LoopingCallDone() +@profiler.trace_cls("rpc") class MagnumPeriodicTasks(periodic_task.PeriodicTasks): '''Magnum periodic Task class diff --git a/magnum/tests/unit/common/test_profiler.py b/magnum/tests/unit/common/test_profiler.py new file mode 100644 index 0000000000..e68cd1ef4c --- /dev/null +++ b/magnum/tests/unit/common/test_profiler.py @@ -0,0 +1,75 @@ +# Copyright 2017 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import inspect +import mock + +from oslo_utils import importutils +from osprofiler import initializer as profiler_init +from osprofiler import opts as profiler_opts +import six.moves as six + +from magnum.common import profiler +from magnum import conf +from magnum.tests import base + + +class TestProfiler(base.TestCase): + def test_all_public_methods_are_traced(self): + profiler_opts.set_defaults(conf.CONF) + self.config(enabled=True, + group='profiler') + + classes = [ + 'magnum.conductor.api.API', + 'magnum.conductor.api.ListenerAPI', + 'magnum.conductor.handlers.ca_conductor.Handler', + 'magnum.conductor.handlers.cluster_conductor.Handler', + 'magnum.conductor.handlers.conductor_listener.Handler', + 'magnum.conductor.handlers.indirection_api.Handler', + 'magnum.service.periodic.MagnumPeriodicTasks', + ] + for clsname in classes: + # give the metaclass and trace_cls() decorator a chance to patch + # methods of the classes above + six.reload_module( + importutils.import_module(clsname.rsplit('.', 1)[0])) + cls = importutils.import_class(clsname) + + for attr, obj in cls.__dict__.items(): + # only public methods are traced + if attr.startswith('_'): + continue + # only checks callables + if not (inspect.ismethod(obj) or inspect.isfunction(obj)): + continue + # osprofiler skips static methods + if isinstance(obj, staticmethod): + continue + + self.assertTrue(getattr(obj, '__traced__', False), obj) + + @mock.patch.object(profiler_init, 'init_from_conf') + def test_setup_profiler(self, mock_init): + self.config(enabled=True, + group='profiler') + + profiler.setup('foo', 'localhost') + + mock_init.assert_called_once_with(conf=conf.CONF, + context=mock.ANY, + project="magnum", + service='foo', + host='localhost') diff --git a/magnum/tests/unit/common/test_rpc.py b/magnum/tests/unit/common/test_rpc.py new file mode 100644 index 0000000000..b7304c4885 --- /dev/null +++ b/magnum/tests/unit/common/test_rpc.py @@ -0,0 +1,252 @@ +# Copyright 2017 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import oslo_messaging as messaging +from oslo_serialization import jsonutils + +from magnum.common import context +from magnum.common import rpc +from magnum.tests import base + + +class TestRpc(base.TestCase): + @mock.patch.object(rpc, 'profiler', None) + @mock.patch.object(rpc, 'RequestContextSerializer') + @mock.patch.object(messaging, 'RPCClient') + def test_get_client(self, mock_client, mock_ser): + rpc.TRANSPORT = mock.Mock() + tgt = mock.Mock() + ser = mock.Mock() + mock_client.return_value = 'client' + mock_ser.return_value = ser + + client = rpc.get_client(tgt, version_cap='1.0', serializer='foo', + timeout=6969) + + mock_ser.assert_called_once_with('foo') + mock_client.assert_called_once_with(rpc.TRANSPORT, + tgt, version_cap='1.0', + serializer=ser, timeout=6969) + self.assertEqual('client', client) + + @mock.patch.object(rpc, 'profiler', mock.Mock()) + @mock.patch.object(rpc, 'ProfilerRequestContextSerializer') + @mock.patch.object(messaging, 'RPCClient') + def test_get_client_profiler_enabled(self, mock_client, mock_ser): + rpc.TRANSPORT = mock.Mock() + tgt = mock.Mock() + ser = mock.Mock() + mock_client.return_value = 'client' + mock_ser.return_value = ser + + client = rpc.get_client(tgt, version_cap='1.0', serializer='foo', + timeout=6969) + + mock_ser.assert_called_once_with('foo') + mock_client.assert_called_once_with(rpc.TRANSPORT, + tgt, version_cap='1.0', + serializer=ser, timeout=6969) + self.assertEqual('client', client) + + @mock.patch.object(rpc, 'profiler', None) + @mock.patch.object(rpc, 'RequestContextSerializer') + @mock.patch.object(messaging, 'get_rpc_server') + def test_get_server(self, mock_get, mock_ser): + rpc.TRANSPORT = mock.Mock() + ser = mock.Mock() + tgt = mock.Mock() + ends = mock.Mock() + mock_ser.return_value = ser + mock_get.return_value = 'server' + + server = rpc.get_server(tgt, ends, serializer='foo') + + mock_ser.assert_called_once_with('foo') + mock_get.assert_called_once_with(rpc.TRANSPORT, tgt, ends, + executor='eventlet', serializer=ser) + self.assertEqual('server', server) + + @mock.patch.object(rpc, 'profiler', mock.Mock()) + @mock.patch.object(rpc, 'ProfilerRequestContextSerializer') + @mock.patch.object(messaging, 'get_rpc_server') + def test_get_server_profiler_enabled(self, mock_get, mock_ser): + rpc.TRANSPORT = mock.Mock() + ser = mock.Mock() + tgt = mock.Mock() + ends = mock.Mock() + mock_ser.return_value = ser + mock_get.return_value = 'server' + + server = rpc.get_server(tgt, ends, serializer='foo') + + mock_ser.assert_called_once_with('foo') + mock_get.assert_called_once_with(rpc.TRANSPORT, tgt, ends, + executor='eventlet', serializer=ser) + self.assertEqual('server', server) + + @mock.patch.object(messaging, 'TransportURL') + def test_get_transport_url(self, mock_url): + conf = mock.Mock() + rpc.CONF = conf + mock_url.parse.return_value = 'foo' + + url = rpc.get_transport_url(url_str='bar') + + self.assertEqual('foo', url) + mock_url.parse.assert_called_once_with(conf, 'bar', + rpc.TRANSPORT_ALIASES) + + @mock.patch.object(messaging, 'TransportURL') + def test_get_transport_url_null(self, mock_url): + conf = mock.Mock() + rpc.CONF = conf + mock_url.parse.return_value = 'foo' + + url = rpc.get_transport_url() + + self.assertEqual('foo', url) + mock_url.parse.assert_called_once_with(conf, None, + rpc.TRANSPORT_ALIASES) + + def test_cleanup_transport_null(self): + rpc.TRANSPORT = None + rpc.NOTIFIER = mock.Mock() + self.assertRaises(AssertionError, rpc.cleanup) + + def test_cleanup_notifier_null(self): + rpc.TRANSPORT = mock.Mock() + rpc.NOTIFIER = None + self.assertRaises(AssertionError, rpc.cleanup) + + def test_cleanup(self): + rpc.NOTIFIER = mock.Mock() + rpc.TRANSPORT = mock.Mock() + trans_cleanup = mock.Mock() + rpc.TRANSPORT.cleanup = trans_cleanup + + rpc.cleanup() + + trans_cleanup.assert_called_once_with() + self.assertIsNone(rpc.TRANSPORT) + self.assertIsNone(rpc.NOTIFIER) + + def test_add_extra_exmods(self): + rpc.EXTRA_EXMODS = [] + + rpc.add_extra_exmods('foo', 'bar') + + self.assertEqual(['foo', 'bar'], rpc.EXTRA_EXMODS) + + def test_clear_extra_exmods(self): + rpc.EXTRA_EXMODS = ['foo', 'bar'] + + rpc.clear_extra_exmods() + + self.assertEqual(0, len(rpc.EXTRA_EXMODS)) + + def test_serialize_entity(self): + with mock.patch.object(jsonutils, 'to_primitive') as mock_prim: + rpc.JsonPayloadSerializer.serialize_entity('context', 'entity') + + mock_prim.assert_called_once_with('entity', convert_instances=True) + + +class TestRequestContextSerializer(base.TestCase): + def setUp(self): + super(TestRequestContextSerializer, self).setUp() + self.mock_base = mock.Mock() + self.ser = rpc.RequestContextSerializer(self.mock_base) + self.ser_null = rpc.RequestContextSerializer(None) + + def test_serialize_entity(self): + self.mock_base.serialize_entity.return_value = 'foo' + + ser_ent = self.ser.serialize_entity('context', 'entity') + + self.mock_base.serialize_entity.assert_called_once_with('context', + 'entity') + self.assertEqual('foo', ser_ent) + + def test_serialize_entity_null_base(self): + ser_ent = self.ser_null.serialize_entity('context', 'entity') + + self.assertEqual('entity', ser_ent) + + def test_deserialize_entity(self): + self.mock_base.deserialize_entity.return_value = 'foo' + + deser_ent = self.ser.deserialize_entity('context', 'entity') + + self.mock_base.deserialize_entity.assert_called_once_with('context', + 'entity') + self.assertEqual('foo', deser_ent) + + def test_deserialize_entity_null_base(self): + deser_ent = self.ser_null.deserialize_entity('context', 'entity') + + self.assertEqual('entity', deser_ent) + + def test_serialize_context(self): + context = mock.Mock() + + self.ser.serialize_context(context) + + context.to_dict.assert_called_once_with() + + @mock.patch.object(context, 'RequestContext') + def test_deserialize_context(self, mock_req): + self.ser.deserialize_context('context') + + mock_req.from_dict.assert_called_once_with('context') + + +class TestProfilerRequestContextSerializer(base.TestCase): + def setUp(self): + super(TestProfilerRequestContextSerializer, self).setUp() + self.ser = rpc.ProfilerRequestContextSerializer(mock.Mock()) + + @mock.patch('magnum.common.rpc.profiler') + def test_serialize_context(self, mock_profiler): + prof = mock_profiler.get.return_value + prof.hmac_key = 'swordfish' + prof.get_base_id.return_value = 'baseid' + prof.get_id.return_value = 'parentid' + + context = mock.Mock() + context.to_dict.return_value = {'project_id': 'test'} + + self.assertEqual({ + 'project_id': 'test', + 'trace_info': { + 'hmac_key': 'swordfish', + 'base_id': 'baseid', + 'parent_id': 'parentid' + } + }, self.ser.serialize_context(context)) + + @mock.patch('magnum.common.rpc.profiler') + def test_deserialize_context(self, mock_profiler): + serialized = {'project_id': 'test', + 'trace_info': { + 'hmac_key': 'swordfish', + 'base_id': 'baseid', + 'parent_id': 'parentid'}} + + context = self.ser.deserialize_context(serialized) + + self.assertEqual('test', context.project_id) + mock_profiler.init.assert_called_once_with( + hmac_key='swordfish', base_id='baseid', parent_id='parentid') diff --git a/setup.cfg b/setup.cfg index bafd2a2b9d..0463141952 100644 --- a/setup.cfg +++ b/setup.cfg @@ -77,3 +77,7 @@ tempest.test_plugins = magnum_tests = magnum.tests.functional.tempest_tests.plugin:MagnumTempestPlugin [wheel] universal = 1 + +[extras] +osprofiler = + osprofiler>=1.4.0 # Apache-2.0 diff --git a/test-requirements.txt b/test-requirements.txt index 975aa9ddac..3deeeb51be 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -15,6 +15,7 @@ mock>=2.0 # BSD openstackdocstheme>=1.5.0 # Apache-2.0 oslosphinx>=4.7.0 # Apache-2.0 oslotest>=1.10.0 # Apache-2.0 +osprofiler>=1.4.0 # Apache-2.0 os-api-ref>=1.0.0 # Apache-2.0 os-testr>=0.8.0 # Apache-2.0 python-subunit>=0.0.18 # Apache-2.0/BSD