Integrate OSprofiler and Trove

*) Add osprofiler wsgi middleware
This middleware is used for 2 things:
1) It checks that the person who wants to trace is trusted and knows
the secret HMAC key.
2) It start tracing in case of proper trace headers
and adds the first wsgi trace point, with info about the HTTP request.

*) Add initialization of osprofiler at start of service
Initialize osprofiler with oslo.messaging notifier which
is used to send notifications to Ceilometer.

*) Use profile enabled context in services rpc interaction
Change context serializer to pass profile context to service;
rpc server will use trace info to initialize their profile context
to continue the profiling for the request as a transaction.

*) Add tracing on service manager and sqlalchemy db engine object

NOTE to test this:
You should put to localrc:
  CEILOMETER_NOTIFICATION_TOPICS=notifications,profiler
  ENABLED_SERVICES+=,ceilometer-acompute,ceilometer-acentral
  ENABLED_SERVICES+=,ceilometer-anotification,ceilometer-collector
  ENABLED_SERVICES+=,ceilometer-alarm-evaluator,ceilometer-alarm-notifier
  ENABLED_SERVICES+=,ceilometer-api

Run any command with --profile <SECRET_KEY>
  $ trove --profile <SECRET_KEY> list
  # it will print <Trace ID>
Get pretty HTML with traces:
  $ osprofiler trace show --html <Trace ID>
Note: Trace showing can be executed with the admin account only.

The change to enable Trove exchange in ceilometer has been merged:
Idce1c327c6d21a767c612c13c1ad52a794017d71 .

The change to enable profile in python-troveclient has been merged:
I5a76e11d428c63d33f6d2c2021426090ebf8340c

We prepared a common BP in oslo-spec as the integration change is
similar in all projects: I95dccdc9f274661767d2659c18b96da169891f30
Currently there are 2 other projects are using osprofiler: Glance &
Cinder, and some others are a work in progress.

Change-Id: I580cce8d2b3c4ec9ce625ac09de6f14e1249f6f5
Signed-off-by: Zhi Yan Liu <zhiyanl@cn.ibm.com>

etc/trove/api-paste.ini

@@ -7,7 +7,7 @@ use = call:trove.common.wsgi:versioned_urlmap
 paste.app_factory = trove.versions:app_factory
 
 [pipeline:troveapi]
-pipeline = faultwrapper authtoken authorization contextwrapper ratelimit extensions troveapp
+pipeline = faultwrapper osprofiler authtoken authorization contextwrapper ratelimit extensions troveapp
 #pipeline = debug extensions troveapp
 
 [filter:extensions]
@@ -28,6 +28,11 @@ paste.filter_factory = trove.common.wsgi:FaultWrapper.factory
 [filter:ratelimit]
 paste.filter_factory = trove.common.limits:RateLimitingMiddleware.factory
 
+[filter:osprofiler]
+paste.filter_factory = osprofiler.web:WsgiMiddleware.factory
+hmac_keys = SECRET_KEY
+enabled = yes
+
 [app:troveapp]
 paste.app_factory = trove.common.api:app_factory
 

etc/trove/trove-conductor.conf.sample

@@ -46,3 +46,9 @@ rabbit_password=f7999d1955c5014aa32c
 
 # The manager class to use for conductor. (string value)
 conductor_manager = trove.conductor.manager.Manager
+
+[profiler]
+# If False fully disable profiling feature.
+#enabled = False
+# If False doesn't trace SQL requests.
+#trace_sqlalchemy = True

etc/trove/trove-guestagent.conf.sample

@@ -105,6 +105,11 @@ root_grant_option = True
 log_dir = /tmp/
 log_file = logfile.txt
 
+[profiler]
+# If False fully disable profiling feature.
+#enabled = False
+# If False doesn't trace SQL requests.
+#trace_sqlalchemy = True
 
 # ========== Datastore Specific Configuration Options ==========
 

etc/trove/trove-taskmanager.conf.sample

@@ -202,6 +202,12 @@ pydev_debug = disabled
 # its own oslo group with defined in it:
 # - tcp_ports; upd_ports;
 
+[profiler]
+# If False fully disable profiling feature.
+#enabled = False
+# If False doesn't trace SQL requests.
+#trace_sqlalchemy = True
+
 [mysql]
 # Format (single port or port range): A, B-C
 # where C greater than B

etc/trove/trove.conf.sample

@@ -194,6 +194,12 @@ api_paste_config = api-paste.ini
 # accessible. The existence of those setting and files will
 # enable SSL.
 
+[profiler]
+# If False fully disable profiling feature.
+#enabled = False
+# If False doesn't trace SQL requests.
+#trace_sqlalchemy = True
+
 [ssl]
 
 #cert_file = /path/to/server.crt

requirements.txt

@@ -38,4 +38,4 @@ six>=1.7.0
 stevedore>=1.1.0  # Apache-2.0
 ordereddict
 oslo.messaging>=1.6.0  # Apache-2.0
-
+osprofiler>=0.3.0                       # Apache-2.0

trove/cmd/api.py

@@ -14,11 +14,13 @@
 #    under the License.
 from oslo_concurrency import processutils
 from trove.cmd.common import with_initialize
+from trove.common import profile
 
 
 @with_initialize
 def main(CONF):
     from trove.common import wsgi
+    profile.setup_profiler('api', CONF.host)
     conf_file = CONF.find_file(CONF.api_paste_config)
     workers = CONF.trove_api_workers or processutils.get_worker_count()
     launcher = wsgi.launch('trove', CONF.bind_port, conf_file,

trove/common/cfg.py

@@ -383,6 +383,18 @@ common_opts = [
                     'become active.'),
 ]
 
+# Profiling specific option groups
+
+profiler_group = cfg.OptGroup(
+    'profiler', title='Profiler options',
+    help="Oslo option group designed for profiler")
+profiler_opts = [
+    cfg.BoolOpt("enabled", default=False,
+                help="If False fully disable profiling feature."),
+    cfg.BoolOpt("trace_sqlalchemy", default=True,
+                help="If False doesn't trace SQL requests.")
+]
+
 # Datastore specific option groups
 
 # Mysql
@@ -764,6 +776,9 @@ CONF = cfg.CONF
 CONF.register_opts(path_opts)
 CONF.register_opts(common_opts)
 
+CONF.register_group(profiler_group)
+CONF.register_opts(profiler_opts, profiler_group)
+
 CONF.register_group(mysql_group)
 CONF.register_group(percona_group)
 CONF.register_group(redis_group)

trove/common/profile.py

@@ -0,0 +1,48 @@
+# Copyright 2015 IBM Corp.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo import messaging
+from osprofiler import notifier
+from osprofiler import web
+
+from trove.common import cfg
+from trove.common import i18n
+from trove.openstack.common import context
+from trove.openstack.common import log as logging
+from trove import rpc
+
+
+_LW = i18n._LW
+LOG = logging.getLogger(__name__)
+CONF = cfg.CONF
+
+
+def setup_profiler(binary, host):
+    if CONF.profiler.enabled:
+        _notifier = notifier.create(
+            "Messaging", messaging, context.get_admin_context().to_dict(),
+            rpc.TRANSPORT, "trove", binary, host)
+        notifier.set(_notifier)
+        LOG.warn(_LW("The OpenStack Profiler is enabled. Using one of the "
+                     "hmac_keys specified in the api-paste.ini file "
+                     "(typically in /etc/trove), a trace can be made of all "
+                     "requests. Only an admin user can retrieve the trace "
+                     "information, however.\n"
+                     "To disable the profiler, add the following to the "
+                     "configuration file:\n"
+                     "[profiler]\n"
+                     "enabled=false"))
+    else:
+        web.disable()

trove/common/rpc/service.py

@@ -20,14 +20,17 @@ import inspect
 import os
 from oslo import messaging
 from oslo.utils import importutils
+from osprofiler import profiler
 from trove.openstack.common.gettextutils import _
 from trove.openstack.common import log as logging
 from trove.openstack.common import loopingcall
 from trove.openstack.common import service
 
 from trove.common import cfg
+from trove.common import profile
 from trove import rpc
 
+
 CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
 
@@ -40,10 +43,12 @@ class RpcService(service.Service):
         self.host = host or CONF.host
         self.binary = binary or os.path.basename(inspect.stack()[-1][1])
         self.topic = topic or self.binary.rpartition('trove-')[2]
-        self.manager_impl = importutils.import_object(manager)
+        _manager = importutils.import_object(manager)
+        self.manager_impl = profiler.trace_cls("rpc")(_manager)
         self.report_interval = CONF.report_interval
         self.rpc_api_version = rpc_api_version or \
             self.manager_impl.RPC_API_VERSION
+        profile.setup_profiler(self.binary, self.host)
 
     def start(self):
         LOG.debug("Creating RPC server for service %s", self.topic)

trove/db/sqlalchemy/session.py

@@ -14,6 +14,8 @@
 #    under the License.
 
 import contextlib
+import osprofiler.sqlalchemy
+import sqlalchemy
 from sqlalchemy import create_engine
 from sqlalchemy import MetaData
 from sqlalchemy.orm import sessionmaker
@@ -77,7 +79,10 @@ def _create_engine(options):
         "echo": CONF.sql_query_log
     }
     LOG.info(_("Creating SQLAlchemy engine with args: %s") % engine_args)
-    return create_engine(options['sql_connection'], **engine_args)
+    db_engine = create_engine(options['sql_connection'], **engine_args)
+    if CONF.profiler.enabled and CONF.profiler.trace_sqlalchemy:
+        osprofiler.sqlalchemy.add_tracing(sqlalchemy, db_engine, "db")
+    return db_engine
 
 
 def get_session(autocommit=True, expire_on_commit=False):

trove/rpc.py

@@ -30,12 +30,16 @@ __all__ = [
     'TRANSPORT_ALIASES',
 ]
 
+
 from oslo.config import cfg
 from oslo import messaging
+from osprofiler import profiler
+
 from trove.common.context import TroveContext
 import trove.common.exception
 from trove.openstack.common import jsonutils
 
+
 CONF = cfg.CONF
 TRANSPORT = None
 NOTIFIER = None
@@ -115,9 +119,21 @@ class RequestContextSerializer(messaging.Serializer):
         return self._base.deserialize_entity(context, entity)
 
     def serialize_context(self, context):
-        return context.to_dict()
+        _context = context.to_dict()
+        prof = profiler.get()
+        if prof:
+            trace_info = {
+                "hmac_key": prof.hmac_key,
+                "base_id": prof.get_base_id(),
+                "parent_id": prof.get_id()
+            }
+            _context.update({"trace_info": trace_info})
+        return _context
 
     def deserialize_context(self, context):
+        trace_info = context.pop("trace_info", None)
+        if trace_info:
+            profiler.init(**trace_info)
         return TroveContext.from_dict(context)