Integrate OSprofiler in Panko

* Add osprofiler wsgi middleware. This middleware is used for 2 things:
  1) It checks that person who wants to trace is trusted and knows
     secret HMAC key.
  2) It starts tracing in case of proper trace headers
     and adds first wsgi trace point, with info about HTTP request

* Traces HTTP API calls
* Traces DB (SQLAlchemy) calls

Demo: https://tovin07.github.io/panko/openstack-event-list.html

HOW TO TEST?

1. Install devstack with panko as usual

2. Install osprofiler

    pip install osprofiler

3. Add these line to /etc/panko/panko.conf

   [profiler]
   enabled = true
   hmac_keys = SECRET_KEY
   connection_string = redis://localhost:6379 # example when using redis

4. Pass HMAC_KEYS to client commands
    - panko client with `--profile <HMAC_KEYS>`
    - openstack client with `--os-profiler <HMAC_KEYS>`

Output will look like this:

To display trace use the command:

  osprofiler trace show --html <TRACE_ID>

5. Use osprofiler to get the trace

    osprofiler trace show \
        --connection-string redis://localhost:6379 \
        --out out.html \
        --html <TRACE_ID>

6. Open that html file with browser and view the result

Change-Id: Ic934acbe1340a3e00361f2709f34725e0e4aa3ba

etc/panko/api_paste.ini

@@ -16,10 +16,10 @@ paste.app_factory = panko.api.app:app_factory
 root = panko.api.controllers.root.VersionsController
 
 [pipeline:pankov2_keystone_pipeline]
-pipeline = cors http_proxy_to_wsgi request_id authtoken pankov2
+pipeline = cors http_proxy_to_wsgi request_id osprofiler authtoken pankov2
 
 [pipeline:pankov2_noauth_pipeline]
-pipeline = cors http_proxy_to_wsgi request_id pankov2
+pipeline = cors http_proxy_to_wsgi request_id osprofiler pankov2
 
 [app:pankov2]
 paste.app_factory = panko.api.app:app_factory
@@ -39,3 +39,7 @@ oslo_config_project = panko
 [filter:http_proxy_to_wsgi]
 paste.filter_factory = oslo_middleware.http_proxy_to_wsgi:HTTPProxyToWSGI.factory
 oslo_config_project = panko
+
+[filter:osprofiler]
+paste.filter_factory = panko.profiler:WsgiMiddleware.factory
+oslo_config_project = panko

panko/api/controllers/v2/capabilities.py

@@ -24,6 +24,7 @@ from wsme import types as wtypes
 import wsmeext.pecan as wsme_pecan
 
 from panko.api.controllers.v2 import base
+from panko import profiler
 from panko import utils
 
 
@@ -31,6 +32,7 @@ def _flatten_capabilities(capabilities):
     return dict((k, v) for k, v in utils.recursive_keypairs(capabilities))
 
 
+@profiler.trace_cls('api')
 class Capabilities(base.Base):
     """A representation of the API and storage capabilities.
 
@@ -53,6 +55,7 @@ class Capabilities(base.Base):
         )
 
 
+@profiler.trace_cls('api')
 class CapabilitiesController(rest.RestController):
     """Manages capabilities queries."""
 

panko/api/controllers/v2/events.py

@@ -33,6 +33,7 @@ from panko.api.controllers.v2 import base
 from panko.api.controllers.v2 import utils as v2_utils
 from panko.api import rbac
 from panko.i18n import _
+from panko import profiler
 from panko import storage
 from panko.storage import models as event_models
 
@@ -234,6 +235,7 @@ def _event_query_to_event_filter(q):
                                admin_proj=admin_proj, **evt_model_filter)
 
 
+@profiler.trace_cls('api')
 class TraitsController(rest.RestController):
     """Works on Event Traits."""
 
@@ -262,6 +264,7 @@ class TraitsController(rest.RestController):
                 for t in pecan.request.conn.get_trait_types(event_type)]
 
 
+@profiler.trace_cls('api')
 class EventTypesController(rest.RestController):
     """Works on Event Types in the system."""
 
@@ -283,6 +286,7 @@ class EventTypesController(rest.RestController):
         return list(pecan.request.conn.get_event_types())
 
 
+@profiler.trace_cls('api')
 class EventsController(rest.RestController):
     """Works on Events."""
 

panko/profiler.py

@@ -0,0 +1,76 @@
+# Copyright 2017 Fujitsu Ltd.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import socket
+
+from oslo_log import log
+from oslo_utils import importutils
+import webob.dec
+
+profiler = importutils.try_import('osprofiler.profiler')
+profiler_initializer = importutils.try_import('osprofiler.initializer')
+profiler_web = importutils.try_import('osprofiler.web')
+
+LOG = log.getLogger(__name__)
+
+
+class WsgiMiddleware(object):
+
+    def __init__(self, application, **kwargs):
+        self.application = application
+
+    @classmethod
+    def factory(cls, global_conf, **local_conf):
+        if profiler_web:
+            return profiler_web.WsgiMiddleware.factory(global_conf)
+
+        def filter_(app):
+            return cls(app)
+
+        return filter_
+
+    @webob.dec.wsgify
+    def __call__(self, request):
+        return request.get_response(self.application)
+
+
+def setup(conf):
+    if hasattr(conf, 'profiler') and conf.profiler.enabled:
+        profiler_initializer.init_from_conf(
+            conf=conf,
+            context={},
+            project=conf.project,
+            service=conf.prog,
+            host=socket.gethostbyname(socket.gethostname()))
+        LOG.info('OSprofiler is enabled.')
+
+
+def trace_cls(name, **kwargs):
+    """Wrap the OSprofiler trace_cls.
+
+    Wrap the OSprofiler trace_cls decorator so that it will not try to
+    patch the class unless OSprofiler is present.
+
+    :param name: The name of action. For example, wsgi, rpc, db, ...
+    :param kwargs: Any other keyword args used by profiler.trace_cls
+    """
+
+    def decorator(cls):
+        if profiler:
+            trace_decorator = profiler.trace_cls(name, **kwargs)
+            return trace_decorator(cls)
+        return cls
+
+    return decorator

panko/service.py

@@ -19,11 +19,15 @@ from oslo_db import options as db_options
 import oslo_i18n
 from oslo_log import log
 from oslo_reports import guru_meditation_report as gmr
+from oslo_utils import importutils
 
 from panko.conf import defaults
 from panko import opts
+from panko import profiler
 from panko import version
 
+profiler_opts = importutils.try_import('osprofiler.opts')
+
 
 def prepare_service(argv=None, config_files=None, share=False):
     conf = cfg.ConfigOpts()
@@ -31,6 +35,8 @@ def prepare_service(argv=None, config_files=None, share=False):
         conf.register_opts(list(options),
                            group=None if group == "DEFAULT" else group)
     db_options.set_defaults(conf)
+    if profiler_opts:
+        profiler_opts.set_defaults(conf)
     if not share:
         defaults.set_cors_middleware_defaults()
         oslo_i18n.enable_lazy()
@@ -44,6 +50,7 @@ def prepare_service(argv=None, config_files=None, share=False):
 
     if not share:
         log.setup(conf, 'panko')
+    profiler.setup(conf)
     # NOTE(liusheng): guru cannot run with service under apache daemon, so when
     # panko-api running with mod_wsgi, the argv is [], we don't start
     # guru.

panko/storage/impl_sqlalchemy.py

@@ -21,6 +21,7 @@ from oslo_db import exception as dbexc
 from oslo_db.sqlalchemy import session as db_session
 from oslo_db.sqlalchemy import utils as oslo_sql_utils
 from oslo_log import log
+from oslo_utils import importutils
 from oslo_utils import timeutils
 import sqlalchemy as sa
 from sqlalchemy.engine import url as sqlalchemy_url
@@ -34,6 +35,8 @@ from panko import utils
 
 LOG = log.getLogger(__name__)
 
+osprofiler_sqlalchemy = importutils.try_import('osprofiler.sqlalchemy')
+
 
 AVAILABLE_CAPABILITIES = {
     'events': {'query': {'simple': True}},
@@ -136,6 +139,10 @@ class Connection(base.Connection):
             options.pop(opt.name, None)
         self._engine_facade = db_session.EngineFacade(self.dress_url(url),
                                                       **options)
+        if osprofiler_sqlalchemy:
+            osprofiler_sqlalchemy.add_tracing(sa,
+                                              self._engine_facade.get_engine(),
+                                              'db')
 
     @staticmethod
     def dress_url(url):