From 13bf7cf6c5c4beb3ddcfae3055af91d0d637d236 Mon Sep 17 00:00:00 2001 From: Shachar Snapiri Date: Wed, 27 Sep 2017 15:15:53 +0300 Subject: [PATCH] Add support for osprofiler in dragonflow Added first version of osprofiler support. Added for all nb_db classes, for all events and first_packet handling in the controller. See osprofiler guide at: https://docs.openstack.org/zaqar/latest/admin/OSprofiler.html Dragonflow instructions are at doc/source/osprofiler.rst. Change-Id: I2ad3405f6884d6a39f243913c1ab5bdaf6855b84 Closes-Bug: #1690272 --- doc/source/osprofiler.rst | 68 ++++++++++++++ dragonflow/common/profiler.py | 88 +++++++++++++++++++ dragonflow/controller/df_bgp_service.py | 5 +- dragonflow/controller/df_config.py | 23 +++++ dragonflow/controller/df_local_controller.py | 6 +- dragonflow/controller/df_publisher_service.py | 5 +- dragonflow/controller/ryu_base_app.py | 5 +- dragonflow/db/model_framework.py | 17 +++- dragonflow/tests/unit/test_ryu_base_app.py | 1 + .../notes/osprofiler-97302fc3440b4fd9.yaml | 12 +++ 10 files changed, 218 insertions(+), 12 deletions(-) create mode 100644 doc/source/osprofiler.rst create mode 100644 dragonflow/common/profiler.py create mode 100644 dragonflow/controller/df_config.py create mode 100644 releasenotes/notes/osprofiler-97302fc3440b4fd9.yaml diff --git a/doc/source/osprofiler.rst b/doc/source/osprofiler.rst new file mode 100644 index 000000000..73aae2797 --- /dev/null +++ b/doc/source/osprofiler.rst @@ -0,0 +1,68 @@ +.. + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +========== +OSProfiler +========== + +OSProfiler provides a tiny but powerful library that is used by +most (soon to be all) OpenStack projects and their python clients. It +provides functionality to be able to generate one trace per request, that goes +through all involved services. This trace can then be extracted and used +to build a tree of calls which can be quite handy for a variety of +reasons (for example in isolating cross-project performance issues). + +More about OSProfiler: +https://docs.openstack.org/osprofiler/latest/ + +Senlin supports using OSProfiler to trace the performance of each +key internal processing, including RESTful API, RPC, cluster actions, +node actions, DB operations etc. + +Enabling OSProfiler +~~~~~~~~~~~~~~~~~~~ + +To configure DevStack to enable OSProfiler, edit the +``${DEVSTACK_DIR}/local.conf`` file and add:: + + enable_plugin panko https://git.openstack.org/openstack/panko + enable_plugin ceilometer https://git.openstack.org/openstack/ceilometer + enable_plugin osprofiler https://git.openstack.org/openstack/osprofiler + +to the ``[[local|localrc]]`` section. + +.. note:: The order of the plugins enabling matters. + +Using OSProfiler +~~~~~~~~~~~~~~~~ + +After successfully deploy your development environment, following profiler +configs will be auto added to ``dragonflow.conf``:: + + [profiler] + enabled = True + trace_sqlalchemy = True + hmac_keys = SECRET_KEY + +``hmac_keys`` is the secret key(s) to use for encrypting context data for +performance profiling, default value is 'SECRET_KEY', you can modify it to +any random string(s). + +Run any command with ``--os-profile SECRET_KEY``:: + + $ openstack --os-profile SECRET_KEY floating ip create public + # it will print a + +Get pretty HTML with traces:: + + $ osprofiler trace show --html diff --git a/dragonflow/common/profiler.py b/dragonflow/common/profiler.py new file mode 100644 index 000000000..96c340c02 --- /dev/null +++ b/dragonflow/common/profiler.py @@ -0,0 +1,88 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import contextlib + +from oslo_log import log as logging + +try: + import osprofiler.initializer + from osprofiler import opts as profiler_opts + from osprofiler import profiler +except Exception: + # osprofiler package is not installed + profiler_opts = None + profiler = None + +from dragonflow import conf as cfg + + +CONF = cfg.CONF +if profiler_opts: + profiler_opts.set_defaults(CONF) +LOG = logging.getLogger(__name__) + + +@contextlib.contextmanager +def profiler_context(*args, **kwargs): + if is_profiler_enabled(): + with profiler.Trace(*args, **kwargs) as tracer: + yield tracer + else: + yield None + + +def _get_profiler_instance(): + # If profiler does not exist or not enabled + if profiler is None or not CONF.profiler.enabled: + return None + instance = profiler.get() + # Try to initialize an instance + if instance is None: + instance = profiler.init(CONF.profiler.hmac_keys) + LOG.debug("Initialized osprofiler, base trace ID: %s", + instance.get_id()) + return instance + + +def is_profiler_enabled(): + return _get_profiler_instance() is not None + + +def setup(name, host='0.0.0.0'): + """Setup OSprofiler notifier and enable profiling. + + :param name: name of the service, that will be profiled + :param host: host (either host name or host address) the service will be + running on. By default host will be set to 0.0.0.0, but more + specified host name / address usage is highly recommended. + """ + if CONF.profiler.enabled: + osprofiler.initializer.init_from_conf( + conf=CONF, + context={}, + project='dragonflow', + service=name, + host=host + ) + LOG.info("OSProfiler is enabled.\n" + "Traces provided from the profiler " + "can only be subscribed to using the same HMAC keys that " + "are configured in Dragonflow's configuration file " + "under the [profiler] section.\n To disable OSprofiler " + "set in /etc/neutron/dragonflow.conf:\n" + "[profiler]\n" + "enabled=False") + + +def get(): + return profiler diff --git a/dragonflow/controller/df_bgp_service.py b/dragonflow/controller/df_bgp_service.py index 2d9aa08dc..c9dbc8ba1 100644 --- a/dragonflow/controller/df_bgp_service.py +++ b/dragonflow/controller/df_bgp_service.py @@ -12,13 +12,13 @@ import sys -from neutron.common import config as common_config from oslo_log import log as logging from oslo_service import loopingcall from oslo_service import service from oslo_utils import importutils from dragonflow import conf as cfg +from dragonflow.controller import df_config from dragonflow.controller import service as df_service from dragonflow.db import api_nb from dragonflow.db import db_store @@ -164,8 +164,7 @@ class BGPService(service.Service): def main(): - common_config.init(sys.argv[1:]) - common_config.setup_logging() + df_config.init(sys.argv) # BGP dynamic route is not a service that needs real time response. # So disable pubsub here and use period task to do BGP job. cfg.CONF.set_override('enable_df_pub_sub', False, group='df') diff --git a/dragonflow/controller/df_config.py b/dragonflow/controller/df_config.py new file mode 100644 index 000000000..a87252cef --- /dev/null +++ b/dragonflow/controller/df_config.py @@ -0,0 +1,23 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from neutron.common import config as common_config + +from dragonflow.common import profiler as df_profiler +from dragonflow import conf as cfg + + +def init(argv): + common_config.init(argv[1:]) + common_config.setup_logging() + df_profiler.setup(argv[0], cfg.CONF.host) diff --git a/dragonflow/controller/df_local_controller.py b/dragonflow/controller/df_local_controller.py index aac03c6b7..357607a0e 100644 --- a/dragonflow/controller/df_local_controller.py +++ b/dragonflow/controller/df_local_controller.py @@ -15,7 +15,6 @@ import sys -from neutron.common import config as common_config from oslo_log import log from oslo_service import loopingcall from ryu.app.ofctl import service as of_service @@ -25,6 +24,7 @@ from ryu import cfg as ryu_cfg from dragonflow.common import utils as df_utils from dragonflow import conf as cfg from dragonflow.controller.common import constants as ctrl_const +from dragonflow.controller import df_config from dragonflow.controller import ryu_base_app from dragonflow.controller import service from dragonflow.controller import topology @@ -343,8 +343,8 @@ def init_ryu_config(): # def main(): chassis_name = cfg.CONF.host - common_config.init(sys.argv[1:]) - common_config.setup_logging() + df_config.init(sys.argv) + init_ryu_config() nb_api = api_nb.NbApi.get_instance(False) controller = DfLocalController(chassis_name, nb_api) diff --git a/dragonflow/controller/df_publisher_service.py b/dragonflow/controller/df_publisher_service.py index 609e77e54..3d13c22bc 100644 --- a/dragonflow/controller/df_publisher_service.py +++ b/dragonflow/controller/df_publisher_service.py @@ -15,12 +15,12 @@ import sys import time import traceback -from neutron.common import config as common_config from oslo_log import log as logging from dragonflow.common import exceptions from dragonflow.common import utils as df_utils from dragonflow import conf as cfg +from dragonflow.controller import df_config from dragonflow.controller import service as df_service from dragonflow.db import api_nb from dragonflow.db import db_common @@ -146,8 +146,7 @@ class PublisherService(object): def main(): - common_config.init(sys.argv[1:]) - common_config.setup_logging() + df_config.init(sys.argv) cfg.CONF.set_override('enable_df_pub_sub', False, group='df') nb_api = api_nb.NbApi.get_instance(False) service = PublisherService(nb_api) diff --git a/dragonflow/controller/ryu_base_app.py b/dragonflow/controller/ryu_base_app.py index 4b13026e6..6e623d069 100644 --- a/dragonflow/controller/ryu_base_app.py +++ b/dragonflow/controller/ryu_base_app.py @@ -25,6 +25,7 @@ from ryu.ofproto import ofproto_parser from ryu.ofproto import ofproto_v1_3 from ryu import utils +from dragonflow.common import profiler as df_profiler from dragonflow.controller.common import constants from dragonflow.controller import dispatcher @@ -133,7 +134,9 @@ class RyuDFAdapter(ofp_handler.OFPHandler): table_id = msg.table_id if table_id in self.table_handlers: handler = self.table_handlers[table_id] - handler(event) + with df_profiler.profiler_context('packet_in', + info={"func": handler.__name__}): + handler(event) else: LOG.info("No handler for table id %(table)s with message " "%(msg)", {'table': table_id, 'msg': msg}) diff --git a/dragonflow/db/model_framework.py b/dragonflow/db/model_framework.py index 3fb09dd2a..155006def 100644 --- a/dragonflow/db/model_framework.py +++ b/dragonflow/db/model_framework.py @@ -20,6 +20,8 @@ from oslo_serialization import jsonutils import six from dragonflow._i18n import _ +from dragonflow.common import profiler as df_profiler + LOG = log.getLogger(__name__) @@ -42,7 +44,7 @@ def _normalize_tuple(v): class _CommonBase(models.Base): '''Base class for extending jsonmodels' Base - Here we add common facilites needed to support: + Here we add common facilities needed to support: * Event registration/dispatch * CRUD hooks @@ -105,7 +107,12 @@ class _CommonBase(models.Base): 'event': event, 'resource': self}) try: - cb(self, *args, **kwargs) + with df_profiler.profiler_context( + 'emit', + info={'func': cb.__name__, + 'module': cb.__module__, + 'event': event}): + cb(self, *args, **kwargs) except Exception: LOG.exception( 'Error while calling %(func)r(*%(_args)r, **%(kw)r)', @@ -340,6 +347,12 @@ def construct_nb_db_model(cls_=None, indexes=None, events=frozenset()): fields = frozenset(n for n, _ in cls_.iterate_over_fields()) cls_._field_names = fields + # Make sure profiler is properly initialized + # if df_profiler.is_profiler_enabled(): + # FIXME snapiri: This SHOULD be the right place, but in our code it + # creates a loop. Should fix to support more traces + # cls_ = df_profiler.get().trace_cls('model')(cls_) + return cls_ if cls_ is None: diff --git a/dragonflow/tests/unit/test_ryu_base_app.py b/dragonflow/tests/unit/test_ryu_base_app.py index 327fcb139..36fe024c2 100644 --- a/dragonflow/tests/unit/test_ryu_base_app.py +++ b/dragonflow/tests/unit/test_ryu_base_app.py @@ -68,6 +68,7 @@ class TestRyuDFAdapter(tests_base.BaseTestCase): self.mock_app.reset_mock() ev = mock.Mock() ev.msg.table_id = 10 + self.mock_app.packet_in_handler.__name__ = 'mock' self.ryu_df_adapter.register_table_handler( 10, self.mock_app.packet_in_handler) self.ryu_df_adapter.OF_packet_in_handler(ev) diff --git a/releasenotes/notes/osprofiler-97302fc3440b4fd9.yaml b/releasenotes/notes/osprofiler-97302fc3440b4fd9.yaml new file mode 100644 index 000000000..4c23174c2 --- /dev/null +++ b/releasenotes/notes/osprofiler-97302fc3440b4fd9.yaml @@ -0,0 +1,12 @@ +--- +features: + - | + Added support for the OSProfiler. + OSProfiler provides a tiny but powerful library that is used by + most (soon to be all) OpenStack projects and their python clients. It + provides functionality to be able to generate one trace per request, that + goes through all involved services. This trace can then be extracted and + used to build a tree of calls which can be quite handy for a variety of + reasons (for example in isolating cross-project performance issues). + The OSProfiler is off by default, extra configuration is required to + enable it. Please refer to the documentation for further details.