Integrate OSprofiler in Aodh

* Add osprofiler wsgi middleware. This middleware is used for 2 things:
  1) It checks that person who wants to trace is trusted and knows
     secret HMAC key.
  2) It starts tracing in case of proper trace headers
     and adds first wsgi trace point, with info about HTTP request

* Traces HTTP API calls
* Traces DB (SQLAlchemy) calls

Demo: https://tovin07.github.io/aodh/alarm_delete.html

HOW TO TEST?

1. Install devstack with aodh as usual

2. Install osprofiler

    pip install osprofiler

3. Add these line to /etc/aodh/aodh.conf

   [profiler]
   enabled = true
   hmac_keys = SECRET_KEY
   connection_string = redis://localhost:6379 # example when using redis

4. Pass HMAC_KEYS to client commands
    - aodh client with `--profile <HMAC_KEYS>`
    - openstack client with `--os-profiler <HMAC_KEYS>`

Output will look like this:

To display trace use the command:

  osprofiler trace show --html <TRACE_ID>

5. Use osprofiler to get the trace

    osprofiler trace show \
        --connection-string redis://localhost:6379 \
        --out out.html \
        --html <TRACE_ID>

6. Open that html file with browser and view the result

Change-Id: I44bf27413af2133451cbd84d908631ce6a2fe1f7

aodh/api/api-paste.ini

@@ -22,10 +22,10 @@ paste.app_factory = aodh.api.app:app_factory
 root = aodh.api.controllers.root.VersionsController
 
 [pipeline:aodhv2_keystone_pipeline]
-pipeline = cors http_proxy_to_wsgi request_id authtoken aodhv2
+pipeline = cors http_proxy_to_wsgi request_id osprofiler authtoken aodhv2
 
 [pipeline:aodhv2_noauth_pipeline]
-pipeline = cors http_proxy_to_wsgi request_id aodhv2
+pipeline = cors http_proxy_to_wsgi request_id osprofiler aodhv2
 
 [app:aodhv2]
 paste.app_factory = aodh.api.app:app_factory
@@ -45,3 +45,7 @@ oslo_config_project = aodh
 [filter:http_proxy_to_wsgi]
 paste.filter_factory = oslo_middleware.http_proxy_to_wsgi:HTTPProxyToWSGI.factory
 oslo_config_project = aodh
+
+[filter:osprofiler]
+paste.filter_factory = aodh.profiler:WsgiMiddleware.factory
+oslo_config_project = aodh

aodh/api/controllers/v2/alarms.py

@@ -46,6 +46,7 @@ from aodh.i18n import _
 from aodh import keystone_client
 from aodh import messaging
 from aodh import notifier
+from aodh import profiler
 from aodh.storage import models
 
 LOG = log.getLogger(__name__)
@@ -185,6 +186,7 @@ ACTIONS_SCHEMA = extension.ExtensionManager(
     notifier.AlarmNotifierService.NOTIFIER_EXTENSIONS_NAMESPACE).names()
 
 
+@profiler.trace_cls('api')
 class Alarm(base.Base):
     """Representation of an alarm."""
 
@@ -532,6 +534,7 @@ def stringify_timestamps(data):
                 for (k, v) in six.iteritems(data))
 
 
+@profiler.trace_cls('api')
 class AlarmController(rest.RestController):
     """Manages operations on a single alarm."""
 
@@ -738,6 +741,7 @@ class AlarmController(rest.RestController):
         return self._enforce_rbac('get_alarm_state').state
 
 
+@profiler.trace_cls('api')
 class AlarmsController(rest.RestController):
     """Manages operations on the alarms collection."""
 

aodh/api/controllers/v2/capabilities.py

@@ -25,6 +25,7 @@ from wsme import types as wtypes
 import wsmeext.pecan as wsme_pecan
 
 from aodh.api.controllers.v2 import base
+from aodh import profiler
 
 
 def _decode_unicode(input):
@@ -90,6 +91,7 @@ class Capabilities(base.Base):
         )
 
 
+@profiler.trace_cls('api')
 class CapabilitiesController(rest.RestController):
     """Manages capabilities queries."""
 

aodh/api/controllers/v2/query.py

@@ -32,6 +32,7 @@ from aodh.api.controllers.v2 import alarms
 from aodh.api.controllers.v2 import base
 from aodh.api import rbac
 from aodh.i18n import _
+from aodh import profiler
 from aodh.storage import models
 
 LOG = log.getLogger(__name__)
@@ -75,6 +76,7 @@ def _list_to_regexp(items, regexp_prefix=""):
     return regexp
 
 
+@profiler.trace_cls('api')
 class ValidatedComplexQuery(object):
     complex_operators = ["and", "or"]
     order_directions = ["asc", "desc"]

aodh/profiler.py

@@ -0,0 +1,76 @@
+# Copyright 2017 Fujitsu Ltd.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import socket
+
+from oslo_log import log
+from oslo_utils import importutils
+import webob.dec
+
+profiler = importutils.try_import('osprofiler.profiler')
+profiler_initializer = importutils.try_import('osprofiler.initializer')
+profiler_web = importutils.try_import('osprofiler.web')
+
+LOG = log.getLogger(__name__)
+
+
+class WsgiMiddleware(object):
+
+    def __init__(self, application, **kwargs):
+        self.application = application
+
+    @classmethod
+    def factory(cls, global_conf, **local_conf):
+        if profiler_web:
+            return profiler_web.WsgiMiddleware.factory(global_conf)
+
+        def filter_(app):
+            return cls(app)
+
+        return filter_
+
+    @webob.dec.wsgify
+    def __call__(self, request):
+        return request.get_response(self.application)
+
+
+def setup(conf):
+    if hasattr(conf, 'profiler') and conf.profiler.enabled:
+        profiler_initializer.init_from_conf(
+            conf=conf,
+            context={},
+            project=conf.project,
+            service=conf.prog,
+            host=socket.gethostbyname(socket.gethostname()))
+        LOG.info('OSprofiler is enabled.')
+
+
+def trace_cls(name, **kwargs):
+    """Wrap the OSprofiler trace_cls.
+
+    Wrap the OSprofiler trace_cls decorator so that it will not try to
+    patch the class unless OSprofiler is present.
+
+    :param name: The name of action. For example, wsgi, rpc, db, ...
+    :param kwargs: Any other keyword args used by profiler.trace_cls
+    """
+
+    def decorator(cls):
+        if profiler:
+            trace_decorator = profiler.trace_cls(name, **kwargs)
+            return trace_decorator(cls)
+        return cls
+
+    return decorator

aodh/service.py

@@ -22,11 +22,14 @@ from oslo_db import options as db_options
 import oslo_i18n
 from oslo_log import log
 from oslo_policy import opts as policy_opts
+from oslo_utils import importutils
 
 from aodh.conf import defaults
 from aodh import keystone_client
 from aodh import messaging
+from aodh import profiler
 
+profiler_opts = importutils.try_import('osprofiler.opts')
 
 OPTS = [
     cfg.IntOpt('http_timeout',
@@ -74,6 +77,8 @@ def prepare_service(argv=None, config_files=None):
     log.set_defaults(default_log_levels=log_levels)
     defaults.set_cors_middleware_defaults()
     db_options.set_defaults(conf)
+    if profiler_opts:
+        profiler_opts.set_defaults(conf)
     policy_opts.set_defaults(conf, policy_file=os.path.abspath(
         os.path.join(os.path.dirname(__file__), "api", "policy.json")))
     from aodh import opts
@@ -88,5 +93,6 @@ def prepare_service(argv=None, config_files=None):
 
     ka_loading.load_auth_from_conf_options(conf, "service_credentials")
     log.setup(conf, 'aodh')
+    profiler.setup(conf)
     messaging.setup()
     return conf

aodh/storage/impl_sqlalchemy.py

@@ -23,8 +23,10 @@ from alembic import migration
 from oslo_db.sqlalchemy import session as db_session
 from oslo_db.sqlalchemy import utils as oslo_sql_utils
 from oslo_log import log
+from oslo_utils import importutils
 from oslo_utils import timeutils
 import six
+import sqlalchemy
 from sqlalchemy import asc
 from sqlalchemy import desc
 from sqlalchemy.engine import url as sqlalchemy_url
@@ -38,6 +40,9 @@ from aodh.storage import models as alarm_api_models
 from aodh.storage.sqlalchemy import models
 from aodh.storage.sqlalchemy import utils as sql_utils
 
+
+osprofiler_sqlalchemy = importutils.try_import('osprofiler.sqlalchemy')
+
 LOG = log.getLogger(__name__)
 
 AVAILABLE_CAPABILITIES = {
@@ -74,6 +79,11 @@ class Connection(base.Connection):
             options.pop(opt.name, None)
         self._engine_facade = db_session.EngineFacade(self.dress_url(url),
                                                       **options)
+
+        if osprofiler_sqlalchemy:
+            osprofiler_sqlalchemy.add_tracing(sqlalchemy,
+                                              self._engine_facade.get_engine(),
+                                              'db')
         self.conf = conf
 
     @staticmethod