From 639e36adbfa0f58ce2c3f31856b4343e9197aa0e Mon Sep 17 00:00:00 2001 From: Dina Belova Date: Fri, 11 Mar 2016 16:29:23 +0300 Subject: [PATCH] Integrate OSprofiler in Keystone OSprofiler is an Oslo library dedicated to enable cross-service OpenStack profiling. This makes possible to trace the OpenStack request through all projects supporting the library, where the profiling is enabled, and generate JSON and HTML human-readable reports, describing what time was spent on which operation, e.g. API or DB request. This change inclues the following: * Add settings for OSprofiler wsgi middleware This middleware is used for 2 things: 1) It checks that person who want to trace is trusted and knows secret HMAC key (that is specified in paste.ini). 2) It initalize profiler in case of proper trace headers and add first wsgi trace point, with info about HTTP request. * Init profiler on start in both cases (httpd or eventlet) * Adding new conf group for profiler (to enable/disable) * By default osprofiler is disabled TBD: * Adding trace point for all DB (sql) calls. For proper format and result oslo.db enginefacade module needs to be updated. To test (old variant, will be changed with new commit to the python-openstackclient): $ . amdminrc # you should be admin user/tenant to fetch profiling data $ keystone --profile SECRET_KEY user-list $ osprofiler trace show --html --out result.html Old python-keystoneclient change: https://review.openstack.org/#/c/114856/ Python-openstack client change: https://review.openstack.org/#/c/255861/ Depends-On: I248b134e0e245bd4cece0ebe225b9b729eedaf78 Change-Id: I3d6eaa7a5ab7ee9ae177f3e1d6cc92f0a01e6a42 --- config-generator/keystone.conf | 1 + etc/keystone-paste.ini | 13 +++++---- keystone/common/config.py | 8 ++++-- keystone/common/profiler.py | 48 ++++++++++++++++++++++++++++++++++ keystone/common/sql/core.py | 3 +++ keystone/server/eventlet.py | 2 ++ keystone/server/wsgi.py | 7 +++++ requirements.txt | 1 + setup.cfg | 2 +- 9 files changed, 77 insertions(+), 8 deletions(-) create mode 100644 keystone/common/profiler.py diff --git a/config-generator/keystone.conf b/config-generator/keystone.conf index fc5ef29cdb..8713c0258d 100644 --- a/config-generator/keystone.conf +++ b/config-generator/keystone.conf @@ -10,6 +10,7 @@ namespace = oslo.policy namespace = oslo.db namespace = oslo.middleware namespace = oslo.service.service +namespace = osprofiler # We don't use oslo.concurrency config options in # keystone now, just in case it slips through unnoticed. #namespace = oslo.concurrency diff --git a/etc/keystone-paste.ini b/etc/keystone-paste.ini index 4f3b0a289c..ec169c06b9 100644 --- a/etc/keystone-paste.ini +++ b/etc/keystone-paste.ini @@ -51,17 +51,17 @@ use = egg:keystone#admin_service [pipeline:public_api] # The last item in this pipeline must be public_service or an equivalent # application. It cannot be a filter. -pipeline = cors sizelimit url_normalize request_id admin_token_auth build_auth_context token_auth json_body ec2_extension public_service +pipeline = cors sizelimit osprofiler url_normalize request_id admin_token_auth build_auth_context token_auth json_body ec2_extension public_service [pipeline:admin_api] # The last item in this pipeline must be admin_service or an equivalent # application. It cannot be a filter. -pipeline = cors sizelimit url_normalize request_id admin_token_auth build_auth_context token_auth json_body ec2_extension s3_extension admin_service +pipeline = cors sizelimit osprofiler url_normalize request_id admin_token_auth build_auth_context token_auth json_body ec2_extension s3_extension admin_service [pipeline:api_v3] # The last item in this pipeline must be service_v3 or an equivalent # application. It cannot be a filter. -pipeline = cors sizelimit url_normalize request_id admin_token_auth build_auth_context token_auth json_body ec2_extension_v3 s3_extension service_v3 +pipeline = cors sizelimit osprofiler url_normalize request_id admin_token_auth build_auth_context token_auth json_body ec2_extension_v3 s3_extension service_v3 [app:public_version_service] use = egg:keystone#public_version_service @@ -70,10 +70,10 @@ use = egg:keystone#public_version_service use = egg:keystone#admin_version_service [pipeline:public_version_api] -pipeline = cors sizelimit url_normalize public_version_service +pipeline = cors sizelimit osprofiler url_normalize public_version_service [pipeline:admin_version_api] -pipeline = cors sizelimit url_normalize admin_version_service +pipeline = cors sizelimit osprofiler url_normalize admin_version_service [composite:main] use = egg:Paste#urlmap @@ -86,3 +86,6 @@ use = egg:Paste#urlmap /v2.0 = admin_api /v3 = api_v3 / = admin_version_api + +[filter:osprofiler] +paste.filter_factory = osprofiler.web:WsgiMiddleware.factory diff --git a/keystone/common/config.py b/keystone/common/config.py index 56f419b6d1..43da36686d 100644 --- a/keystone/common/config.py +++ b/keystone/common/config.py @@ -20,6 +20,7 @@ from oslo_config import cfg from oslo_log import log import oslo_messaging from oslo_middleware import cors +from osprofiler import opts as profiler import passlib.utils from keystone import exception @@ -1228,7 +1229,7 @@ def list_opts(): return list(FILE_OPTIONS.items()) -def set_middleware_defaults(): +def set_external_opts_defaults(): """Update default configuration options for oslo.middleware.""" # CORS Defaults # TODO(krotscheck): Update with https://review.openstack.org/#/c/285368/ @@ -1252,8 +1253,11 @@ def set_middleware_defaults(): 'PATCH'] ) + # configure OSprofiler options + profiler.set_defaults(CONF, enabled=False, trace_sqlalchemy=False) + def set_config_defaults(): """Override all configuration default values for keystone.""" set_default_for_default_log_levels() - set_middleware_defaults() + set_external_opts_defaults() diff --git a/keystone/common/profiler.py b/keystone/common/profiler.py new file mode 100644 index 0000000000..a1bddffb5c --- /dev/null +++ b/keystone/common/profiler.py @@ -0,0 +1,48 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + +import oslo_messaging +import osprofiler.notifier +import osprofiler.web + +from keystone.common import config +from keystone.i18n import _LI + + +CONF = config.CONF +LOG = logging.getLogger(__name__) + + +def setup(name, host='0.0.0.0'): # nosec + """Setup OSprofiler notifier and enable profiling. + + :param name: name of the service that will be profiled + :param host: hostname or host IP address that the service will be + running on. By default host will be set to 0.0.0.0, but more + specified host name / address usage is highly recommended. + """ + if CONF.profiler.enabled: + _notifier = osprofiler.notifier.create( + "Messaging", oslo_messaging, {}, + oslo_messaging.get_transport(CONF), "keystone", name, host) + osprofiler.notifier.set(_notifier) + osprofiler.web.enable(CONF.profiler.hmac_keys) + LOG.info(_LI("OSProfiler is enabled.\n" + "Traces provided from the profiler " + "can only be subscribed to using the same HMAC keys that " + "are configured in Keystone's configuration file " + "under the [profiler] section. \n To disable OSprofiler " + "set in /etc/keystone/keystone.conf:\n" + "[profiler]\n" + "enabled=false")) diff --git a/keystone/common/sql/core.py b/keystone/common/sql/core.py index 0d3821485e..f9d4f1bee1 100644 --- a/keystone/common/sql/core.py +++ b/keystone/common/sql/core.py @@ -169,6 +169,9 @@ _main_context_manager = None def _get_main_context_manager(): + # TODO(DinaBelova): add DB profiling + # this requires oslo.db modification for proper format and functionality + # will be done in Newton timeframe global _main_context_manager if not _main_context_manager: diff --git a/keystone/server/eventlet.py b/keystone/server/eventlet.py index e688baed86..5156ec864d 100644 --- a/keystone/server/eventlet.py +++ b/keystone/server/eventlet.py @@ -34,6 +34,7 @@ oslo_i18n.enable_lazy() from keystone.common import config from keystone.common import environment +from keystone.common import profiler from keystone.common import utils from keystone.i18n import _ from keystone.server import common @@ -65,6 +66,7 @@ def create_server(conf, name, host, port, workers): server = environment.Server(app, host=host, port=port, keepalive=CONF.eventlet_server.tcp_keepalive, keepidle=CONF.eventlet_server.tcp_keepidle) + profiler.setup(name, host) if CONF.eventlet_server_ssl.enable: server.set_ssl(CONF.eventlet_server_ssl.certfile, CONF.eventlet_server_ssl.keyfile, diff --git a/keystone/server/wsgi.py b/keystone/server/wsgi.py index a62a84603a..256e936df7 100644 --- a/keystone/server/wsgi.py +++ b/keystone/server/wsgi.py @@ -17,6 +17,8 @@ import logging from oslo_config import cfg import oslo_i18n +from keystone.common import profiler + # NOTE(dstanek): i18n.enable_lazy() must be called before # keystone.i18n._() is called to ensure it has the desired lazy lookup @@ -51,6 +53,11 @@ def initialize_application(name, post_log_configured_function=lambda: None): _unused, application = common.setup_backends( startup_application_fn=loadapp) + + # setup OSprofiler notifier and enable the profiling if that is configured + # in Keystone configuration file. + profiler.setup(name) + return application diff --git a/requirements.txt b/requirements.txt index 8ebcc71b47..b4880d29da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,3 +37,4 @@ dogpile.cache>=0.5.7 # BSD jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT pycadf!=2.0.0,>=1.1.0 # Apache-2.0 msgpack-python>=0.4.0 # Apache-2.0 +osprofiler>=1.1.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index 03242e2882..cd3e712df5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -177,7 +177,7 @@ oslo.config.opts = keystone.notifications = keystone.notifications:list_opts oslo.config.opts.defaults = - keystone = keystone.common.config:set_middleware_defaults + keystone = keystone.common.config:set_external_opts_defaults paste.filter_factory = admin_token_auth = keystone.middleware:AdminTokenAuthMiddleware.factory