Integrate OSprofiler and Manila

*) Add osprofiler wsgi middleware

This middleware is used for 2 things:
1) It checks that person who want to trace is trusted and knows
secret HMAC key.
2) It start tracing in case of proper trace headers and add
first wsgi trace point, with info about HTTP request
*) Add initialization of osprofiler at start of serivce.

You should use python-manilaclient with this patch:

https://review.opendev.org/#/c/769731

Run any command with --profile SECRET_KEY

  $ manila --profile SECRET_KEY create NFS 1 --name <share> \
    --description <share_description> --share-network <network> \
    --share-type default
  # it will print <Trace ID>

Get pretty HTML with traces:

  $ osprofiler trace show --html <Trace ID> --connection-string \
   <connection_string> --out <output.html>
  e.g. --connection-string can be redis://localhost:6379

Note that osprofiler should be run from admin user name & tenant.

Implements: blueprint manila-os-profiler
Change-Id: I3bce1f04d1cfebfacd78ed135a949a068c78987d
This commit is contained in:
kpdev 2021-01-07 14:13:45 +01:00
parent 8c85cf31a5
commit fbc2b4d289
10 changed files with 114 additions and 10 deletions

View File

@ -11,16 +11,16 @@ use = call:manila.api:root_app_factory
[composite:openstack_share_api]
use = call:manila.api.middleware.auth:pipeline_factory
noauth = cors faultwrap http_proxy_to_wsgi sizelimit noauth api
keystone = cors faultwrap http_proxy_to_wsgi sizelimit authtoken keystonecontext api
keystone_nolimit = cors faultwrap http_proxy_to_wsgi sizelimit authtoken keystonecontext api
noauth = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler noauth api
keystone = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler authtoken keystonecontext api
keystone_nolimit = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler authtoken keystonecontext api
[composite:openstack_share_api_v2]
use = call:manila.api.middleware.auth:pipeline_factory
noauth = cors faultwrap http_proxy_to_wsgi sizelimit noauth apiv2
noauthv2 = cors faultwrap http_proxy_to_wsgi sizelimit noauthv2 apiv2
keystone = cors faultwrap http_proxy_to_wsgi sizelimit authtoken keystonecontext apiv2
keystone_nolimit = cors faultwrap http_proxy_to_wsgi sizelimit authtoken keystonecontext apiv2
noauth = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler noauth apiv2
noauthv2 = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler noauthv2 apiv2
keystone = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler authtoken keystonecontext apiv2
keystone_nolimit = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler authtoken keystonecontext apiv2
[filter:faultwrap]
paste.filter_factory = manila.api.middleware.fault:FaultWrapper.factory
@ -34,6 +34,9 @@ paste.filter_factory = manila.api.middleware.auth:NoAuthMiddlewarev2_60.factory
[filter:sizelimit]
paste.filter_factory = oslo_middleware.sizelimit:RequestBodySizeLimiter.factory
[filter:osprofiler]
paste.filter_factory = osprofiler.web:WsgiMiddleware.factory
[filter:http_proxy_to_wsgi]
paste.filter_factory = oslo_middleware.http_proxy_to_wsgi:HTTPProxyToWSGI.factory

View File

@ -72,6 +72,7 @@ oslo.service==2.4.0
oslo.upgradecheck==1.3.0
oslo.utils==4.7.0
oslotest==4.4.1
osprofiler==3.4.0
packaging==20.4
paramiko==2.7.2
Paste==3.4.3

View File

@ -23,6 +23,7 @@ import datetime
from functools import wraps
import ipaddress
import sys
import threading
import warnings
# NOTE(uglide): Required to override default oslo_db Query class
@ -37,8 +38,10 @@ from oslo_db.sqlalchemy import session
from oslo_db.sqlalchemy import utils as db_utils
from oslo_log import log
from oslo_utils import excutils
from oslo_utils import importutils
from oslo_utils import timeutils
from oslo_utils import uuidutils
import sqlalchemy
from sqlalchemy import MetaData
from sqlalchemy import or_
from sqlalchemy.orm import joinedload
@ -54,7 +57,11 @@ from manila import exception
from manila.i18n import _
from manila import quota
osprofiler_sqlalchemy = importutils.try_import('osprofiler.sqlalchemy')
CONF = cfg.CONF
CONF.import_group("profiler", "manila.service")
LOG = log.getLogger(__name__)
QUOTAS = quota.QUOTAS
@ -63,6 +70,8 @@ _DEFAULT_QUOTA_NAME = 'default'
PER_PROJECT_QUOTAS = []
_FACADE = None
_LOCK = threading.Lock()
_DEFAULT_SQL_CONNECTION = 'sqlite://'
db_options.set_defaults(cfg.CONF,
@ -70,9 +79,15 @@ db_options.set_defaults(cfg.CONF,
def _create_facade_lazily():
global _FACADE
global _LOCK, _FACADE
if _FACADE is None:
_FACADE = session.EngineFacade.from_config(cfg.CONF)
with _LOCK:
if _FACADE is None:
_FACADE = session.EngineFacade.from_config(cfg.CONF)
if CONF.profiler.enabled and CONF.profiler.trace_sqlalchemy:
osprofiler_sqlalchemy.add_tracing(sqlalchemy,
_FACADE.get_engine(),
"db")
return _FACADE

View File

@ -29,6 +29,8 @@ from oslo_config import cfg
import oslo_messaging as messaging
from oslo_messaging.rpc import dispatcher
from oslo_serialization import jsonutils
from oslo_utils import importutils
profiler = importutils.try_import('osprofiler.profiler')
import manila.context
import manila.exception
@ -115,9 +117,24 @@ class RequestContextSerializer(messaging.Serializer):
return self._base.deserialize_entity(context, entity)
def serialize_context(self, context):
return context.to_dict()
_context = context.to_dict()
if profiler is not None:
prof = profiler.get()
if prof:
trace_info = {
"hmac_key": prof.hmac_key,
"base_id": prof.get_base_id(),
"parent_id": prof.get_id()
}
_context.update({"trace_info": trace_info})
return _context
def deserialize_context(self, context):
trace_info = context.pop("trace_info", None)
if trace_info:
if profiler is not None:
profiler.init(**trace_info)
return manila.context.RequestContext.from_dict(context)

View File

@ -36,6 +36,10 @@ from manila import exception
from manila import rpc
from manila import version
osprofiler_initializer = importutils.try_import('osprofiler.initializer')
profiler = importutils.try_import('osprofiler.profiler')
profiler_opts = importutils.try_import('osprofiler.opts')
LOG = log.getLogger(__name__)
service_opts = [
@ -68,6 +72,26 @@ service_opts = [
CONF = cfg.CONF
CONF.register_opts(service_opts)
if profiler_opts:
profiler_opts.set_defaults(CONF)
def setup_profiler(binary, host):
if (osprofiler_initializer is None or
profiler is None or
profiler_opts is None):
LOG.debug('osprofiler is not present')
return
if CONF.profiler.enabled:
osprofiler_initializer.init_from_conf(
conf=CONF,
context=context.get_admin_context().to_dict(),
project="manila",
service=binary,
host=host
)
LOG.warning("OSProfiler is enabled.")
class Service(service.Service):
@ -89,6 +113,10 @@ class Service(service.Service):
self.topic = topic
self.manager_class_name = manager
manager_class = importutils.import_class(self.manager_class_name)
if CONF.profiler.enabled and profiler is not None:
manager_class = profiler.trace_cls("rpc")(manager_class)
self.service = None
self.manager = manager_class(host=self.host,
service_name=service_name,
*args, **kwargs)
@ -100,6 +128,8 @@ class Service(service.Service):
self.timers = []
self.coordinator = coordination
setup_profiler(binary, host)
def start(self):
version_string = version.version_string()
LOG.info('Starting %(topic)s node (version %(version_string)s)',
@ -311,6 +341,8 @@ class WSGIService(service.ServiceBase):
"greater than 1. Input value ignored.", {'name': name})
# Reset workers to default
self.workers = None
setup_profiler(name, self.host)
self.server = wsgi.Server(
CONF,
name,

View File

@ -55,6 +55,8 @@ from manila.share import snapshot_access
from manila.share import utils as share_utils
from manila import utils
profiler = importutils.try_import('osprofiler.profiler')
LOG = log.getLogger(__name__)
share_manager_opts = [
@ -259,6 +261,8 @@ class ShareManager(manager.SchedulerDependentManager):
self.message_api = message_api.API()
self.share_api = api.API()
if CONF.profiler.enabled and profiler is not None:
self.driver = profiler.trace_cls("driver")(self.driver)
self.hooks = []
self._init_hook_drivers()

View File

@ -153,12 +153,27 @@ class TestCase(base_test.BaseTestCase):
fake_notifier.stub_notifier(self)
self._disable_osprofiler()
# Locks must be cleaned up after tests
CONF.set_override('backend_url', 'file://' + lock_path,
group='coordination')
coordination.LOCK_COORDINATOR.start()
self.addCleanup(coordination.LOCK_COORDINATOR.stop)
def _disable_osprofiler(self):
"""Disable osprofiler.
osprofiler should not run for unit tests.
"""
def side_effect(value):
return value
mock_decorator = mock.MagicMock(side_effect=side_effect)
p = mock.patch("osprofiler.profiler.trace_cls",
return_value=mock_decorator)
p.start()
def tearDown(self):
"""Runs after each test method to tear down test environment."""
super(TestCase, self).tearDown()

View File

@ -22,6 +22,7 @@ from oslo_service import wsgi
# Need to register global_opts
from manila.common import config
from manila import rpc
from manila import service
from manila import version
CONF = cfg.CONF
@ -33,4 +34,5 @@ def initialize_application():
config.verify_share_protocols()
log.setup(CONF, "manila")
rpc.init(CONF)
service.setup_profiler("manila-api", CONF.host)
return wsgi.Loader(CONF).load_app(name='osapi_share')

View File

@ -0,0 +1,14 @@
---
features:
- |
OSprofiler support was introduced. To allow its usage the api-paste.ini
file needs to be modified to contain osprofiler middleware. Also
`[profiler]` section needs to be added to the manila.conf file with
`enabled`, `hmac_keys` and `trace_sqlalchemy` flags defined.
security:
- OSprofiler support requires passing of trace information
between various OpenStack services. This information is
securely signed by one of HMAC keys, defined in manila.conf configuration
file. To allow cross-project tracing user should use the key, that is
common among all OpenStack services they want to trace.

View File

@ -25,6 +25,7 @@ oslo.service>=2.4.0 # Apache-2.0
oslo.upgradecheck>=1.3.0 # Apache-2.0
oslo.utils>=4.7.0 # Apache-2.0
oslo.concurrency>=4.3.0 # Apache-2.0
osprofiler>=3.4.0 # Apache-2.0
paramiko>=2.7.2 # LGPLv2.1+
Paste>=3.4.3 # MIT
PasteDeploy>=2.1.0 # MIT