Integrate OSprofiler in Keystone

OSprofiler is an Oslo library dedicated to enable cross-service
OpenStack profiling. This makes possible to trace the OpenStack
request through all projects supporting the library, where the
profiling is enabled, and generate JSON and HTML human-readable
reports, describing what time was spent on which operation, e.g.
API or DB request.

This change inclues the following:

* Add settings for OSprofiler wsgi middleware
  This middleware is used for 2 things:
  1) It checks that person who want to trace is trusted and knows
     secret HMAC key (that is specified in paste.ini).
  2) It initalize profiler in case of proper trace headers
     and add first wsgi trace point, with info about HTTP request.
* Init profiler on start in both cases (httpd or eventlet)
* Adding new conf group for profiler (to enable/disable)
* By default osprofiler is disabled

TBD:
* Adding trace point for all DB (sql) calls. For proper format and
result oslo.db enginefacade module needs to be updated.

To test (old variant, will be changed with new commit to the
python-openstackclient):

$ . amdminrc # you should be admin user/tenant to fetch profiling data
$ keystone --profile SECRET_KEY user-list
$ osprofiler trace show --html --out result.html <TRACE_ID>

Old python-keystoneclient change: https://review.openstack.org/#/c/114856/
Python-openstack client change: https://review.openstack.org/#/c/255861/

Depends-On: I248b134e0e245bd4cece0ebe225b9b729eedaf78
Change-Id: I3d6eaa7a5ab7ee9ae177f3e1d6cc92f0a01e6a42

config-generator/keystone.conf

@@ -10,6 +10,7 @@ namespace = oslo.policy
 namespace = oslo.db
 namespace = oslo.middleware
 namespace = oslo.service.service
+namespace = osprofiler
 # We don't use oslo.concurrency config options in
 # keystone now, just in case it slips through unnoticed.
 #namespace = oslo.concurrency

etc/keystone-paste.ini

@@ -51,17 +51,17 @@ use = egg:keystone#admin_service
 [pipeline:public_api]
 # The last item in this pipeline must be public_service or an equivalent
 # application. It cannot be a filter.
-pipeline = cors sizelimit url_normalize request_id admin_token_auth build_auth_context token_auth json_body ec2_extension public_service
+pipeline = cors sizelimit osprofiler url_normalize request_id admin_token_auth build_auth_context token_auth json_body ec2_extension public_service
 
 [pipeline:admin_api]
 # The last item in this pipeline must be admin_service or an equivalent
 # application. It cannot be a filter.
-pipeline = cors sizelimit url_normalize request_id admin_token_auth build_auth_context token_auth json_body ec2_extension s3_extension admin_service
+pipeline = cors sizelimit osprofiler url_normalize request_id admin_token_auth build_auth_context token_auth json_body ec2_extension s3_extension admin_service
 
 [pipeline:api_v3]
 # The last item in this pipeline must be service_v3 or an equivalent
 # application. It cannot be a filter.
-pipeline = cors sizelimit url_normalize request_id admin_token_auth build_auth_context token_auth json_body ec2_extension_v3 s3_extension service_v3
+pipeline = cors sizelimit osprofiler url_normalize request_id admin_token_auth build_auth_context token_auth json_body ec2_extension_v3 s3_extension service_v3
 
 [app:public_version_service]
 use = egg:keystone#public_version_service
@@ -70,10 +70,10 @@ use = egg:keystone#public_version_service
 use = egg:keystone#admin_version_service
 
 [pipeline:public_version_api]
-pipeline = cors sizelimit url_normalize public_version_service
+pipeline = cors sizelimit osprofiler url_normalize public_version_service
 
 [pipeline:admin_version_api]
-pipeline = cors sizelimit url_normalize admin_version_service
+pipeline = cors sizelimit osprofiler url_normalize admin_version_service
 
 [composite:main]
 use = egg:Paste#urlmap
@@ -86,3 +86,6 @@ use = egg:Paste#urlmap
 /v2.0 = admin_api
 /v3 = api_v3
 / = admin_version_api
+
+[filter:osprofiler]
+paste.filter_factory = osprofiler.web:WsgiMiddleware.factory

keystone/common/config.py

@@ -20,6 +20,7 @@ from oslo_config import cfg
 from oslo_log import log
 import oslo_messaging
 from oslo_middleware import cors
+from osprofiler import opts as profiler
 import passlib.utils
 
 from keystone import exception
@@ -1228,7 +1229,7 @@ def list_opts():
     return list(FILE_OPTIONS.items())
 
 
-def set_middleware_defaults():
+def set_external_opts_defaults():
     """Update default configuration options for oslo.middleware."""
     # CORS Defaults
     # TODO(krotscheck): Update with https://review.openstack.org/#/c/285368/
@@ -1252,8 +1253,11 @@ def set_middleware_defaults():
                                     'PATCH']
                      )
 
+    # configure OSprofiler options
+    profiler.set_defaults(CONF, enabled=False, trace_sqlalchemy=False)
+
 
 def set_config_defaults():
     """Override all configuration default values for keystone."""
     set_default_for_default_log_levels()
-    set_middleware_defaults()
+    set_external_opts_defaults()

keystone/common/profiler.py

@@ -0,0 +1,48 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging
+
+import oslo_messaging
+import osprofiler.notifier
+import osprofiler.web
+
+from keystone.common import config
+from keystone.i18n import _LI
+
+
+CONF = config.CONF
+LOG = logging.getLogger(__name__)
+
+
+def setup(name, host='0.0.0.0'):  # nosec
+    """Setup OSprofiler notifier and enable profiling.
+
+    :param name: name of the service that will be profiled
+    :param host: hostname or host IP address that the service will be
+                 running on. By default host will be set to 0.0.0.0, but more
+                 specified host name / address usage is highly recommended.
+    """
+    if CONF.profiler.enabled:
+        _notifier = osprofiler.notifier.create(
+            "Messaging", oslo_messaging, {},
+            oslo_messaging.get_transport(CONF), "keystone", name, host)
+        osprofiler.notifier.set(_notifier)
+        osprofiler.web.enable(CONF.profiler.hmac_keys)
+        LOG.info(_LI("OSProfiler is enabled.\n"
+                     "Traces provided from the profiler "
+                     "can only be subscribed to using the same HMAC keys that "
+                     "are configured in Keystone's configuration file "
+                     "under the [profiler] section. \n To disable OSprofiler "
+                     "set in /etc/keystone/keystone.conf:\n"
+                     "[profiler]\n"
+                     "enabled=false"))

keystone/common/sql/core.py

@@ -169,6 +169,9 @@ _main_context_manager = None
 
 
 def _get_main_context_manager():
+    # TODO(DinaBelova): add DB profiling
+    # this requires oslo.db modification for proper format and functionality
+    # will be done in Newton timeframe
     global _main_context_manager
 
     if not _main_context_manager:

keystone/server/eventlet.py

@@ -34,6 +34,7 @@ oslo_i18n.enable_lazy()
 
 from keystone.common import config
 from keystone.common import environment
+from keystone.common import profiler
 from keystone.common import utils
 from keystone.i18n import _
 from keystone.server import common
@@ -65,6 +66,7 @@ def create_server(conf, name, host, port, workers):
     server = environment.Server(app, host=host, port=port,
                                 keepalive=CONF.eventlet_server.tcp_keepalive,
                                 keepidle=CONF.eventlet_server.tcp_keepidle)
+    profiler.setup(name, host)
     if CONF.eventlet_server_ssl.enable:
         server.set_ssl(CONF.eventlet_server_ssl.certfile,
                        CONF.eventlet_server_ssl.keyfile,

keystone/server/wsgi.py

@@ -17,6 +17,8 @@ import logging
 from oslo_config import cfg
 import oslo_i18n
 
+from keystone.common import profiler
+
 
 # NOTE(dstanek): i18n.enable_lazy() must be called before
 # keystone.i18n._() is called to ensure it has the desired lazy lookup
@@ -51,6 +53,11 @@ def initialize_application(name, post_log_configured_function=lambda: None):
 
     _unused, application = common.setup_backends(
         startup_application_fn=loadapp)
+
+    # setup OSprofiler notifier and enable the profiling if that is configured
+    # in Keystone configuration file.
+    profiler.setup(name)
+
     return application
 
 

requirements.txt

@@ -37,3 +37,4 @@ dogpile.cache>=0.5.7 # BSD
 jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT
 pycadf!=2.0.0,>=1.1.0 # Apache-2.0
 msgpack-python>=0.4.0 # Apache-2.0
+osprofiler>=1.1.0 # Apache-2.0

setup.cfg

@@ -177,7 +177,7 @@ oslo.config.opts =
     keystone.notifications = keystone.notifications:list_opts
 
 oslo.config.opts.defaults =
-    keystone = keystone.common.config:set_middleware_defaults
+    keystone = keystone.common.config:set_external_opts_defaults
 
 paste.filter_factory =
     admin_token_auth = keystone.middleware:AdminTokenAuthMiddleware.factory