From fbc2b4d2896d90feaa6be2f67764688b8e571ca3 Mon Sep 17 00:00:00 2001 From: kpdev Date: Thu, 7 Jan 2021 14:13:45 +0100 Subject: [PATCH] Integrate OSprofiler and Manila *) Add osprofiler wsgi middleware This middleware is used for 2 things: 1) It checks that person who want to trace is trusted and knows secret HMAC key. 2) It start tracing in case of proper trace headers and add first wsgi trace point, with info about HTTP request *) Add initialization of osprofiler at start of serivce. You should use python-manilaclient with this patch: https://review.opendev.org/#/c/769731 Run any command with --profile SECRET_KEY $ manila --profile SECRET_KEY create NFS 1 --name \ --description --share-network \ --share-type default # it will print Get pretty HTML with traces: $ osprofiler trace show --html --connection-string \ --out e.g. --connection-string can be redis://localhost:6379 Note that osprofiler should be run from admin user name & tenant. Implements: blueprint manila-os-profiler Change-Id: I3bce1f04d1cfebfacd78ed135a949a068c78987d --- etc/manila/api-paste.ini | 17 ++++++---- lower-constraints.txt | 1 + manila/db/sqlalchemy/api.py | 19 +++++++++-- manila/rpc.py | 19 ++++++++++- manila/service.py | 32 +++++++++++++++++++ manila/share/manager.py | 4 +++ manila/test.py | 15 +++++++++ manila/wsgi/wsgi.py | 2 ++ ...ntegrate-os-profiler-b637041861029175.yaml | 14 ++++++++ requirements.txt | 1 + 10 files changed, 114 insertions(+), 10 deletions(-) create mode 100644 releasenotes/notes/bp-integrate-os-profiler-b637041861029175.yaml diff --git a/etc/manila/api-paste.ini b/etc/manila/api-paste.ini index 477f1de1a1..c47803b23a 100644 --- a/etc/manila/api-paste.ini +++ b/etc/manila/api-paste.ini @@ -11,16 +11,16 @@ use = call:manila.api:root_app_factory [composite:openstack_share_api] use = call:manila.api.middleware.auth:pipeline_factory -noauth = cors faultwrap http_proxy_to_wsgi sizelimit noauth api -keystone = cors faultwrap http_proxy_to_wsgi sizelimit authtoken keystonecontext api -keystone_nolimit = cors faultwrap http_proxy_to_wsgi sizelimit authtoken keystonecontext api +noauth = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler noauth api +keystone = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler authtoken keystonecontext api +keystone_nolimit = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler authtoken keystonecontext api [composite:openstack_share_api_v2] use = call:manila.api.middleware.auth:pipeline_factory -noauth = cors faultwrap http_proxy_to_wsgi sizelimit noauth apiv2 -noauthv2 = cors faultwrap http_proxy_to_wsgi sizelimit noauthv2 apiv2 -keystone = cors faultwrap http_proxy_to_wsgi sizelimit authtoken keystonecontext apiv2 -keystone_nolimit = cors faultwrap http_proxy_to_wsgi sizelimit authtoken keystonecontext apiv2 +noauth = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler noauth apiv2 +noauthv2 = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler noauthv2 apiv2 +keystone = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler authtoken keystonecontext apiv2 +keystone_nolimit = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler authtoken keystonecontext apiv2 [filter:faultwrap] paste.filter_factory = manila.api.middleware.fault:FaultWrapper.factory @@ -34,6 +34,9 @@ paste.filter_factory = manila.api.middleware.auth:NoAuthMiddlewarev2_60.factory [filter:sizelimit] paste.filter_factory = oslo_middleware.sizelimit:RequestBodySizeLimiter.factory +[filter:osprofiler] +paste.filter_factory = osprofiler.web:WsgiMiddleware.factory + [filter:http_proxy_to_wsgi] paste.filter_factory = oslo_middleware.http_proxy_to_wsgi:HTTPProxyToWSGI.factory diff --git a/lower-constraints.txt b/lower-constraints.txt index dbc4daecfd..57f22de724 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -72,6 +72,7 @@ oslo.service==2.4.0 oslo.upgradecheck==1.3.0 oslo.utils==4.7.0 oslotest==4.4.1 +osprofiler==3.4.0 packaging==20.4 paramiko==2.7.2 Paste==3.4.3 diff --git a/manila/db/sqlalchemy/api.py b/manila/db/sqlalchemy/api.py index 3e5c9377bd..f759a137cc 100644 --- a/manila/db/sqlalchemy/api.py +++ b/manila/db/sqlalchemy/api.py @@ -23,6 +23,7 @@ import datetime from functools import wraps import ipaddress import sys +import threading import warnings # NOTE(uglide): Required to override default oslo_db Query class @@ -37,8 +38,10 @@ from oslo_db.sqlalchemy import session from oslo_db.sqlalchemy import utils as db_utils from oslo_log import log from oslo_utils import excutils +from oslo_utils import importutils from oslo_utils import timeutils from oslo_utils import uuidutils +import sqlalchemy from sqlalchemy import MetaData from sqlalchemy import or_ from sqlalchemy.orm import joinedload @@ -54,7 +57,11 @@ from manila import exception from manila.i18n import _ from manila import quota + +osprofiler_sqlalchemy = importutils.try_import('osprofiler.sqlalchemy') + CONF = cfg.CONF +CONF.import_group("profiler", "manila.service") LOG = log.getLogger(__name__) QUOTAS = quota.QUOTAS @@ -63,6 +70,8 @@ _DEFAULT_QUOTA_NAME = 'default' PER_PROJECT_QUOTAS = [] _FACADE = None +_LOCK = threading.Lock() + _DEFAULT_SQL_CONNECTION = 'sqlite://' db_options.set_defaults(cfg.CONF, @@ -70,9 +79,15 @@ db_options.set_defaults(cfg.CONF, def _create_facade_lazily(): - global _FACADE + global _LOCK, _FACADE if _FACADE is None: - _FACADE = session.EngineFacade.from_config(cfg.CONF) + with _LOCK: + if _FACADE is None: + _FACADE = session.EngineFacade.from_config(cfg.CONF) + if CONF.profiler.enabled and CONF.profiler.trace_sqlalchemy: + osprofiler_sqlalchemy.add_tracing(sqlalchemy, + _FACADE.get_engine(), + "db") return _FACADE diff --git a/manila/rpc.py b/manila/rpc.py index 8f11d5ea16..2a4ebaa53f 100644 --- a/manila/rpc.py +++ b/manila/rpc.py @@ -29,6 +29,8 @@ from oslo_config import cfg import oslo_messaging as messaging from oslo_messaging.rpc import dispatcher from oslo_serialization import jsonutils +from oslo_utils import importutils +profiler = importutils.try_import('osprofiler.profiler') import manila.context import manila.exception @@ -115,9 +117,24 @@ class RequestContextSerializer(messaging.Serializer): return self._base.deserialize_entity(context, entity) def serialize_context(self, context): - return context.to_dict() + _context = context.to_dict() + if profiler is not None: + prof = profiler.get() + if prof: + trace_info = { + "hmac_key": prof.hmac_key, + "base_id": prof.get_base_id(), + "parent_id": prof.get_id() + } + _context.update({"trace_info": trace_info}) + return _context def deserialize_context(self, context): + trace_info = context.pop("trace_info", None) + if trace_info: + if profiler is not None: + profiler.init(**trace_info) + return manila.context.RequestContext.from_dict(context) diff --git a/manila/service.py b/manila/service.py index 9e4c16f4fa..2fcf759e62 100644 --- a/manila/service.py +++ b/manila/service.py @@ -36,6 +36,10 @@ from manila import exception from manila import rpc from manila import version +osprofiler_initializer = importutils.try_import('osprofiler.initializer') +profiler = importutils.try_import('osprofiler.profiler') +profiler_opts = importutils.try_import('osprofiler.opts') + LOG = log.getLogger(__name__) service_opts = [ @@ -68,6 +72,26 @@ service_opts = [ CONF = cfg.CONF CONF.register_opts(service_opts) +if profiler_opts: + profiler_opts.set_defaults(CONF) + + +def setup_profiler(binary, host): + if (osprofiler_initializer is None or + profiler is None or + profiler_opts is None): + LOG.debug('osprofiler is not present') + return + + if CONF.profiler.enabled: + osprofiler_initializer.init_from_conf( + conf=CONF, + context=context.get_admin_context().to_dict(), + project="manila", + service=binary, + host=host + ) + LOG.warning("OSProfiler is enabled.") class Service(service.Service): @@ -89,6 +113,10 @@ class Service(service.Service): self.topic = topic self.manager_class_name = manager manager_class = importutils.import_class(self.manager_class_name) + if CONF.profiler.enabled and profiler is not None: + manager_class = profiler.trace_cls("rpc")(manager_class) + + self.service = None self.manager = manager_class(host=self.host, service_name=service_name, *args, **kwargs) @@ -100,6 +128,8 @@ class Service(service.Service): self.timers = [] self.coordinator = coordination + setup_profiler(binary, host) + def start(self): version_string = version.version_string() LOG.info('Starting %(topic)s node (version %(version_string)s)', @@ -311,6 +341,8 @@ class WSGIService(service.ServiceBase): "greater than 1. Input value ignored.", {'name': name}) # Reset workers to default self.workers = None + setup_profiler(name, self.host) + self.server = wsgi.Server( CONF, name, diff --git a/manila/share/manager.py b/manila/share/manager.py index aee2dbc0fb..aed0f6eccd 100644 --- a/manila/share/manager.py +++ b/manila/share/manager.py @@ -55,6 +55,8 @@ from manila.share import snapshot_access from manila.share import utils as share_utils from manila import utils +profiler = importutils.try_import('osprofiler.profiler') + LOG = log.getLogger(__name__) share_manager_opts = [ @@ -259,6 +261,8 @@ class ShareManager(manager.SchedulerDependentManager): self.message_api = message_api.API() self.share_api = api.API() + if CONF.profiler.enabled and profiler is not None: + self.driver = profiler.trace_cls("driver")(self.driver) self.hooks = [] self._init_hook_drivers() diff --git a/manila/test.py b/manila/test.py index 6fe00ec43e..4c2b96dd2d 100644 --- a/manila/test.py +++ b/manila/test.py @@ -153,12 +153,27 @@ class TestCase(base_test.BaseTestCase): fake_notifier.stub_notifier(self) + self._disable_osprofiler() + # Locks must be cleaned up after tests CONF.set_override('backend_url', 'file://' + lock_path, group='coordination') coordination.LOCK_COORDINATOR.start() self.addCleanup(coordination.LOCK_COORDINATOR.stop) + def _disable_osprofiler(self): + """Disable osprofiler. + + osprofiler should not run for unit tests. + """ + + def side_effect(value): + return value + mock_decorator = mock.MagicMock(side_effect=side_effect) + p = mock.patch("osprofiler.profiler.trace_cls", + return_value=mock_decorator) + p.start() + def tearDown(self): """Runs after each test method to tear down test environment.""" super(TestCase, self).tearDown() diff --git a/manila/wsgi/wsgi.py b/manila/wsgi/wsgi.py index 4640c95396..131ef0d330 100644 --- a/manila/wsgi/wsgi.py +++ b/manila/wsgi/wsgi.py @@ -22,6 +22,7 @@ from oslo_service import wsgi # Need to register global_opts from manila.common import config from manila import rpc +from manila import service from manila import version CONF = cfg.CONF @@ -33,4 +34,5 @@ def initialize_application(): config.verify_share_protocols() log.setup(CONF, "manila") rpc.init(CONF) + service.setup_profiler("manila-api", CONF.host) return wsgi.Loader(CONF).load_app(name='osapi_share') diff --git a/releasenotes/notes/bp-integrate-os-profiler-b637041861029175.yaml b/releasenotes/notes/bp-integrate-os-profiler-b637041861029175.yaml new file mode 100644 index 0000000000..a964232090 --- /dev/null +++ b/releasenotes/notes/bp-integrate-os-profiler-b637041861029175.yaml @@ -0,0 +1,14 @@ +--- +features: + - | + OSprofiler support was introduced. To allow its usage the api-paste.ini + file needs to be modified to contain osprofiler middleware. Also + `[profiler]` section needs to be added to the manila.conf file with + `enabled`, `hmac_keys` and `trace_sqlalchemy` flags defined. +security: + - OSprofiler support requires passing of trace information + between various OpenStack services. This information is + securely signed by one of HMAC keys, defined in manila.conf configuration + file. To allow cross-project tracing user should use the key, that is + common among all OpenStack services they want to trace. + diff --git a/requirements.txt b/requirements.txt index 8ca9379303..b9b71ca5e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,6 +25,7 @@ oslo.service>=2.4.0 # Apache-2.0 oslo.upgradecheck>=1.3.0 # Apache-2.0 oslo.utils>=4.7.0 # Apache-2.0 oslo.concurrency>=4.3.0 # Apache-2.0 +osprofiler>=3.4.0 # Apache-2.0 paramiko>=2.7.2 # LGPLv2.1+ Paste>=3.4.3 # MIT PasteDeploy>=2.1.0 # MIT