diff --git a/etc/api-paste.ini b/etc/api-paste.ini index 5902651634c..934ef3ff9a5 100644 --- a/etc/api-paste.ini +++ b/etc/api-paste.ini @@ -32,3 +32,6 @@ paste.app_factory = neutron.api.versions:Versions.factory [app:neutronapiapp_v2_0] paste.app_factory = neutron.api.v2.router:APIRouter.factory + +[filter:osprofiler] +paste.filter_factory = osprofiler.web:WsgiMiddleware.factory diff --git a/neutron/common/profiler.py b/neutron/common/profiler.py new file mode 100644 index 00000000000..4e463921c6f --- /dev/null +++ b/neutron/common/profiler.py @@ -0,0 +1,48 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from oslo_log import log as logging +import oslo_messaging +import osprofiler.notifier +from osprofiler import opts as profiler_opts +import osprofiler.web + +from neutron._i18n import _LI + +CONF = cfg.CONF +profiler_opts.set_defaults(CONF) +LOG = logging.getLogger(__name__) + + +def setup(name, host='0.0.0.0'): # nosec + """Setup OSprofiler notifier and enable profiling. + + :param name: name of the service, that will be profiled + :param host: host (either host name or host address) the service will be + running on. By default host will be set to 0.0.0.0, but more + specified host name / address usage is highly recommended. + """ + if CONF.profiler.enabled: + _notifier = osprofiler.notifier.create( + "Messaging", oslo_messaging, {}, + oslo_messaging.get_transport(CONF), "neutron", name, host) + osprofiler.notifier.set(_notifier) + osprofiler.web.enable(CONF.profiler.hmac_keys) + LOG.info(_LI("OSProfiler is enabled.\n" + "Traces provided from the profiler " + "can only be subscribed to using the same HMAC keys that " + "are configured in Neutron's configuration file " + "under the [profiler] section.\n To disable OSprofiler " + "set in /etc/neutron/neutron.conf:\n" + "[profiler]\n" + "enabled=false")) diff --git a/neutron/common/rpc.py b/neutron/common/rpc.py index c179813889b..bf8ec82c5bf 100644 --- a/neutron/common/rpc.py +++ b/neutron/common/rpc.py @@ -25,6 +25,7 @@ import oslo_messaging from oslo_messaging import serializer as om_serializer from oslo_service import service from oslo_utils import excutils +from osprofiler import profiler from neutron._i18n import _LE, _LW from neutron.common import exceptions @@ -212,10 +213,22 @@ class RequestContextSerializer(om_serializer.Serializer): return self._base.deserialize_entity(ctxt, entity) def serialize_context(self, ctxt): - return ctxt.to_dict() + _context = ctxt.to_dict() + prof = profiler.get() + if prof: + trace_info = { + "hmac_key": prof.hmac_key, + "base_id": prof.get_base_id(), + "parent_id": prof.get_id() + } + _context['trace_info'] = trace_info + return _context def deserialize_context(self, ctxt): rpc_ctxt_dict = ctxt.copy() + trace_info = rpc_ctxt_dict.pop("trace_info", None) + if trace_info: + profiler.init(**trace_info) user_id = rpc_ctxt_dict.pop('user_id', None) if not user_id: user_id = rpc_ctxt_dict.pop('user', None) @@ -225,6 +238,7 @@ class RequestContextSerializer(om_serializer.Serializer): return context.Context(user_id, tenant_id, **rpc_ctxt_dict) +@profiler.trace_cls("rpc") class Service(service.Service): """Service object for binaries running on hosts. diff --git a/neutron/db/api.py b/neutron/db/api.py index b721c35b2f8..1e6731f1b76 100644 --- a/neutron/db/api.py +++ b/neutron/db/api.py @@ -23,6 +23,8 @@ from oslo_db import exception as db_exc from oslo_db.sqlalchemy import enginefacade from oslo_utils import excutils from oslo_utils import uuidutils +import osprofiler.sqlalchemy +import sqlalchemy from neutron.db import common_db_mixin @@ -58,6 +60,11 @@ def _create_facade_lazily(): context_manager.configure(sqlite_fk=True, **cfg.CONF.database) _FACADE = context_manager._factory.get_legacy_facade() + if cfg.CONF.profiler.enabled and cfg.CONF.profiler.trace_sqlalchemy: + osprofiler.sqlalchemy.add_tracing(sqlalchemy, + _FACADE.get_engine(), + "db") + return _FACADE diff --git a/neutron/manager.py b/neutron/manager.py index 03873e1829d..061a4d3563f 100644 --- a/neutron/manager.py +++ b/neutron/manager.py @@ -19,6 +19,7 @@ from oslo_config import cfg from oslo_log import log as logging import oslo_messaging from oslo_service import periodic_task +from osprofiler import profiler import six from neutron._i18n import _, _LI @@ -31,7 +32,13 @@ LOG = logging.getLogger(__name__) CORE_PLUGINS_NAMESPACE = 'neutron.core_plugins' +class ManagerMeta(profiler.TracedMeta, type(periodic_task.PeriodicTasks)): + pass + + +@six.add_metaclass(ManagerMeta) class Manager(periodic_task.PeriodicTasks): + __trace_args__ = {"name": "rpc"} # Set RPC API version to 1.0 by default. target = oslo_messaging.Target(version='1.0') @@ -86,6 +93,7 @@ def validate_pre_plugin_load(): return msg +@six.add_metaclass(profiler.TracedMeta) class NeutronManager(object): """Neutron's Manager class. @@ -95,6 +103,7 @@ class NeutronManager(object): The caller should make sure that NeutronManager is a singleton. """ _instance = None + __trace_args__ = {"name": "rpc"} def __init__(self, options=None, config_file=None): # If no options have been provided, create an empty dict diff --git a/neutron/pecan_wsgi/app.py b/neutron/pecan_wsgi/app.py index e5976f285c9..7fa2a0af47b 100644 --- a/neutron/pecan_wsgi/app.py +++ b/neutron/pecan_wsgi/app.py @@ -88,10 +88,12 @@ def _wrap_app(app): app.set_latent( allow_headers=['X-Auth-Token', 'X-Identity-Status', 'X-Roles', 'X-Service-Catalog', 'X-User-Id', 'X-Tenant-Id', - 'X-OpenStack-Request-ID'], + 'X-OpenStack-Request-ID', + 'X-Trace-Info', 'X-Trace-HMAC'], allow_methods=['GET', 'PUT', 'POST', 'DELETE', 'PATCH'], expose_headers=['X-Auth-Token', 'X-Subject-Token', 'X-Service-Token', - 'X-OpenStack-Request-ID'] + 'X-OpenStack-Request-ID', + 'X-Trace-Info', 'X-Trace-HMAC'] ) return app diff --git a/neutron/plugins/ml2/drivers/agent/_common_agent.py b/neutron/plugins/ml2/drivers/agent/_common_agent.py index 0e0263d586a..fd836235eef 100644 --- a/neutron/plugins/ml2/drivers/agent/_common_agent.py +++ b/neutron/plugins/ml2/drivers/agent/_common_agent.py @@ -23,6 +23,7 @@ from oslo_config import cfg from oslo_log import log as logging from oslo_service import loopingcall from oslo_service import service +from osprofiler import profiler from neutron._i18n import _LE, _LI from neutron.agent.l2.extensions import manager as ext_manager @@ -39,6 +40,7 @@ from neutron.plugins.ml2.drivers.agent import config as cagt_config # noqa LOG = logging.getLogger(__name__) +@profiler.trace_cls("rpc") class CommonAgentLoop(service.Service): def __init__(self, manager, polling_interval, diff --git a/neutron/plugins/ml2/drivers/linuxbridge/agent/linuxbridge_neutron_agent.py b/neutron/plugins/ml2/drivers/linuxbridge/agent/linuxbridge_neutron_agent.py index 479b3c654f7..940fd32b539 100644 --- a/neutron/plugins/ml2/drivers/linuxbridge/agent/linuxbridge_neutron_agent.py +++ b/neutron/plugins/ml2/drivers/linuxbridge/agent/linuxbridge_neutron_agent.py @@ -37,6 +37,7 @@ from neutron.agent.linux import utils from neutron.agent import securitygroups_rpc as sg_rpc from neutron.common import config as common_config from neutron.common import exceptions +from neutron.common import profiler as setup_profiler from neutron.common import topics from neutron.common import utils as n_utils from neutron.plugins.common import constants as p_const @@ -928,6 +929,7 @@ def main(): agent = ca.CommonAgentLoop(manager, polling_interval, quitting_rpc_timeout, constants.AGENT_TYPE_LINUXBRIDGE, LB_AGENT_BINARY) + setup_profiler.setup("neutron-linuxbridge-agent", cfg.CONF.host) LOG.info(_LI("Agent initialized successfully, now running... ")) launcher = service.launch(cfg.CONF, agent) launcher.wait() diff --git a/neutron/plugins/ml2/drivers/mech_sriov/agent/sriov_nic_agent.py b/neutron/plugins/ml2/drivers/mech_sriov/agent/sriov_nic_agent.py index 1ff8d154511..5ec0a7d7485 100644 --- a/neutron/plugins/ml2/drivers/mech_sriov/agent/sriov_nic_agent.py +++ b/neutron/plugins/ml2/drivers/mech_sriov/agent/sriov_nic_agent.py @@ -25,6 +25,7 @@ from oslo_config import cfg from oslo_log import log as logging import oslo_messaging from oslo_service import loopingcall +from osprofiler import profiler import six from neutron._i18n import _, _LE, _LI, _LW @@ -33,6 +34,7 @@ from neutron.agent import rpc as agent_rpc from neutron.agent import securitygroups_rpc as sg_rpc from neutron.api.rpc.callbacks import resources from neutron.common import config as common_config +from neutron.common import profiler as setup_profiler from neutron.common import topics from neutron.common import utils as n_utils from neutron import context @@ -103,6 +105,7 @@ class SriovNicSwitchRpcCallbacks(sg_rpc.SecurityGroupAgentRpcCallbackMixin): self.agent.updated_devices.add(port_data['device']) +@profiler.trace_cls("rpc") class SriovNicSwitchAgent(object): def __init__(self, physical_devices_mappings, exclude_devices, polling_interval): @@ -458,5 +461,6 @@ def main(): LOG.exception(_LE("Agent Initialization Failed")) raise SystemExit(1) # Start everything. + setup_profiler.setup("neutron-sriov-nic-agent", cfg.CONF.host) LOG.info(_LI("Agent initialized successfully, now running... ")) agent.daemon_loop() diff --git a/neutron/plugins/ml2/drivers/openvswitch/agent/main.py b/neutron/plugins/ml2/drivers/openvswitch/agent/main.py index 2fd965274ec..87564991c1b 100644 --- a/neutron/plugins/ml2/drivers/openvswitch/agent/main.py +++ b/neutron/plugins/ml2/drivers/openvswitch/agent/main.py @@ -22,6 +22,7 @@ from oslo_log import log as logging from oslo_utils import importutils from neutron.common import config as common_config +from neutron.common import profiler from neutron.common import utils as n_utils @@ -46,4 +47,5 @@ def main(): mod.init_config() common_config.setup_logging() n_utils.log_opt_values(LOG) + profiler.setup("neutron-ovs-agent", cfg.CONF.host) mod.main() diff --git a/neutron/plugins/ml2/drivers/openvswitch/agent/ovs_dvr_neutron_agent.py b/neutron/plugins/ml2/drivers/openvswitch/agent/ovs_dvr_neutron_agent.py index 842f58f5f22..4c5bd434846 100644 --- a/neutron/plugins/ml2/drivers/openvswitch/agent/ovs_dvr_neutron_agent.py +++ b/neutron/plugins/ml2/drivers/openvswitch/agent/ovs_dvr_neutron_agent.py @@ -20,6 +20,7 @@ from oslo_config import cfg from oslo_log import log as logging import oslo_messaging from oslo_utils import excutils +from osprofiler import profiler from neutron._i18n import _LE, _LI, _LW from neutron.common import utils as n_utils @@ -112,6 +113,7 @@ class OVSPort(object): return self.ofport +@profiler.trace_cls("ovs_dvr_agent") class OVSDVRNeutronAgent(object): ''' Implements OVS-based DVR(Distributed Virtual Router), for overlay networks. diff --git a/neutron/plugins/ml2/drivers/openvswitch/agent/ovs_neutron_agent.py b/neutron/plugins/ml2/drivers/openvswitch/agent/ovs_neutron_agent.py index b31744c7ec8..876d2372720 100644 --- a/neutron/plugins/ml2/drivers/openvswitch/agent/ovs_neutron_agent.py +++ b/neutron/plugins/ml2/drivers/openvswitch/agent/ovs_neutron_agent.py @@ -28,6 +28,7 @@ from oslo_log import log as logging import oslo_messaging from oslo_service import loopingcall from oslo_service import systemd +from osprofiler import profiler import six from six import moves @@ -98,6 +99,7 @@ def has_zero_prefixlen_address(ip_addresses): return any(netaddr.IPNetwork(ip).prefixlen == 0 for ip in ip_addresses) +@profiler.trace_cls("rpc") class OVSNeutronAgent(sg_rpc.SecurityGroupAgentRpcCallbackMixin, l2population_rpc.L2populationRpcCallBackTunnelMixin, dvr_rpc.DVRAgentRpcCallbackMixin): diff --git a/neutron/service.py b/neutron/service.py index e7a77eb10cc..d0038dbc55c 100644 --- a/neutron/service.py +++ b/neutron/service.py @@ -28,6 +28,7 @@ from oslo_utils import importutils from neutron._i18n import _LE, _LI from neutron.common import config +from neutron.common import profiler from neutron.common import rpc as n_rpc from neutron.conf import service from neutron import context @@ -64,6 +65,9 @@ class WsgiService(object): class NeutronApiService(WsgiService): """Class for neutron-api service.""" + def __init__(self, app_name): + profiler.setup('neutron-server', cfg.CONF.host) + super(NeutronApiService, self).__init__(app_name) @classmethod def create(cls, app_name='neutron'): @@ -246,6 +250,7 @@ class Service(n_rpc.Service): self.periodic_fuzzy_delay = periodic_fuzzy_delay self.saved_args, self.saved_kwargs = args, kwargs self.timers = [] + profiler.setup(binary, host) super(Service, self).__init__(host, topic, manager=self.manager) def start(self): diff --git a/neutron/tests/contrib/gate_hook.sh b/neutron/tests/contrib/gate_hook.sh index e953ab2edd7..46ec936784a 100644 --- a/neutron/tests/contrib/gate_hook.sh +++ b/neutron/tests/contrib/gate_hook.sh @@ -61,6 +61,7 @@ case $VENV in "api"|"api-pecan"|"full-pecan"|"dsvm-scenario") load_rc_hook api_extensions load_rc_hook qos + load_conf_hook osprofiler if [[ "$VENV" =~ "pecan" ]]; then load_conf_hook pecan fi diff --git a/neutron/tests/contrib/hooks/osprofiler b/neutron/tests/contrib/hooks/osprofiler new file mode 100644 index 00000000000..117d0f29f85 --- /dev/null +++ b/neutron/tests/contrib/hooks/osprofiler @@ -0,0 +1,7 @@ +[[post-config|/etc/neutron/api-paste.ini]] + +[composite:neutronapi_v2_0] +use = call:neutron.auth:pipeline_factory +noauth = cors request_id catch_errors osprofiler extensions neutronapiapp_v2_0 +keystone = cors request_id catch_errors osprofiler authtoken keystonecontext extensions neutronapiapp_v2_0 + diff --git a/releasenotes/notes/add-osprofiler-support-7fc2de3001187075.yaml b/releasenotes/notes/add-osprofiler-support-7fc2de3001187075.yaml new file mode 100644 index 00000000000..29b95baf29f --- /dev/null +++ b/releasenotes/notes/add-osprofiler-support-7fc2de3001187075.yaml @@ -0,0 +1,32 @@ +--- +fixes: + - Missing OSprofiler support was added. This cross-project profiling library + allows to trace various OpenStack requests through all OpenStack + services that support it. To initiate OpenStack + request tracing `--profile ` option needs to be added to + the CLI command. This key needs to present one of the secret keys + defined in neutron.conf configuration file with `hmac_keys` option + under the `[profiler]` configuration section. To enable or disable + Neutron profiling the appropriate `enabled` option under the same + section needs to be set either to `True` or `False`. By default + Neutron will trace all API and RPC requests, but there is an opportunity + to trace DB requests as well. For this purpose `trace_sqlalchemy` + option needs to be set to `True`. As a prerequisite OSprofiler + library and its storage backend needs to be installed to the + environment. If so (and if profiling is enabled in neutron.conf) + the trace can be generated via command - + `$ neutron --profile SECRET_KEY `. + At the end of output there will be message with , and + to plot nice HTML graphs the following command should be used - + `$ osprofiler trace show --html --out result.html` +upgrade: + - OSprofiler support was introduced. To allow its usage the api-paste.ini + file needs to be modified to contain osprofiler middleware. Also + `[profiler]` section needs to be added to the neutron.conf file with + `enabled`, `hmac_keys` and `trace_sqlalchemy` flags defined. +security: + - OSprofiler support requires passing of trace information + between various OpenStack services. This information is + securely signed by one of HMAC keys, defined in neutron.conf configuration + file. To allow cross-project tracing user should use the key, that is + common among all OpenStack services he or she wants to trace. diff --git a/requirements.txt b/requirements.txt index efd47c5986a..f40c8c4ab41 100644 --- a/requirements.txt +++ b/requirements.txt @@ -42,6 +42,7 @@ oslo.serialization>=1.10.0 # Apache-2.0 oslo.service>=1.10.0 # Apache-2.0 oslo.utils>=3.5.0 # Apache-2.0 oslo.versionedobjects!=1.9.0,>=1.5.0 # Apache-2.0 +osprofiler>=1.3.0 # Apache-2.0 ovs>=2.5.0;python_version=='2.7' # Apache-2.0 ovs>=2.6.0.dev1;python_version>='3.4' # Apache-2.0