Integrate OSProfiler and Nova

* Add osprofiler wsgi middleware. This middleware is used for 2 things:
  1) It checks that person who want to trace is trusted and knows
     secret HMAC key.
  2) It starts tracing in case of proper trace headers
     and adds the first wsgi trace point with info about the HTTP request

* Add initialization of osprofiler on start of a service
  Currently that includes oslo.messaging notifier instance creation
  to send Ceilometer backend notifications.

oslo-spec: https://review.openstack.org/#/c/103825/
python-novaclient change: https://review.openstack.org/#/c/254699/
based on: https://review.openstack.org/#/c/105096/

Co-Authored-By: Boris Pavlovic <boris@pavlovic.me>
Co-Authored-By: Munoz, Obed N <obed.n.munoz@intel.com>
Co-Authored-By: Roman Podoliaka <rpodolyaka@mirantis.com>
Co-Authored-By: Tovin Seven <vinhnt@vn.fujitsu.com>

Implements: blueprint osprofiler-support-in-nova
Change-Id: I82d2badc8c1fcec27c3fce7c3c20e0f3b76414f1

nova/db/sqlalchemy/api.py

@@ -31,6 +31,7 @@ from oslo_db.sqlalchemy import update_match
 from oslo_db.sqlalchemy import utils as sqlalchemyutils
 from oslo_log import log as logging
 from oslo_utils import excutils
+from oslo_utils import importutils
 from oslo_utils import timeutils
 from oslo_utils import uuidutils
 import six
@@ -68,6 +69,7 @@ from nova.i18n import _, _LI, _LE, _LW
 from nova import quota
 from nova import safe_utils
 
+profiler_sqlalchemy = importutils.try_import('osprofiler.sqlalchemy')
 
 CONF = nova.conf.CONF
 
@@ -110,6 +112,14 @@ def configure(conf):
     main_context_manager.configure(**_get_db_conf(conf.database))
     api_context_manager.configure(**_get_db_conf(conf.api_database))
 
+    if profiler_sqlalchemy and CONF.profiler.enabled \
+            and CONF.profiler.trace_sqlalchemy:
+
+        main_context_manager.append_on_engine_create(
+            lambda eng: profiler_sqlalchemy.add_tracing(sa, eng, "db"))
+        api_context_manager.append_on_engine_create(
+            lambda eng: profiler_sqlalchemy.add_tracing(sa, eng, "db"))
+
 
 def create_context_manager(connection=None):
     """Create a database context manager object.

nova/utils.py

@@ -60,6 +60,8 @@ from nova.i18n import _, _LE, _LI, _LW
 import nova.network
 from nova import safe_utils
 
+profiler = importutils.try_import('osprofiler.profiler')
+
 
 CONF = nova.conf.CONF
 
@@ -1045,6 +1047,22 @@ def validate_integer(value, name, min_value=None, max_value=None):
     return value
 
 
+def _serialize_profile_info():
+    if not profiler:
+        return None
+    prof = profiler.get()
+    trace_info = None
+    if prof:
+        # FIXME(DinaBelova): we'll add profiler.get_info() method
+        # to extract this info -> we'll need to update these lines
+        trace_info = {
+            "hmac_key": prof.hmac_key,
+            "base_id": prof.get_base_id(),
+            "parent_id": prof.get_id()
+        }
+    return trace_info
+
+
 def spawn(func, *args, **kwargs):
     """Passthrough method for eventlet.spawn.
 
@@ -1056,6 +1074,7 @@ def spawn(func, *args, **kwargs):
     context when using this method to spawn a new thread.
     """
     _context = common_context.get_current()
+    profiler_info = _serialize_profile_info()
 
     @functools.wraps(func)
     def context_wrapper(*args, **kwargs):
@@ -1063,6 +1082,8 @@ def spawn(func, *args, **kwargs):
         # available for the logger to pull from threadlocal storage.
         if _context is not None:
             _context.update_store()
+        if profiler_info and profiler:
+            profiler.init(**profiler_info)
         return func(*args, **kwargs)
 
     return eventlet.spawn(context_wrapper, *args, **kwargs)
@@ -1079,6 +1100,7 @@ def spawn_n(func, *args, **kwargs):
     context when using this method to spawn a new thread.
     """
     _context = common_context.get_current()
+    profiler_info = _serialize_profile_info()
 
     @functools.wraps(func)
     def context_wrapper(*args, **kwargs):
@@ -1086,6 +1108,8 @@ def spawn_n(func, *args, **kwargs):
         # available for the logger to pull from threadlocal storage.
         if _context is not None:
             _context.update_store()
+        if profiler_info and profiler:
+            profiler.init(**profiler_info)
         func(*args, **kwargs)
 
     eventlet.spawn_n(context_wrapper, *args, **kwargs)

setup.cfg

@@ -218,6 +218,10 @@ output_file = nova/locale/nova.pot
 [wheel]
 universal = 1
 
+[extras]
+osprofiler =
+    osprofiler>=1.4.0  # Apache-2.0
+
 [pbr]
 # Treat sphinx warnings as errors during the docs build; this helps us keep
 # the documentation clean.

test-requirements.txt

@@ -19,6 +19,7 @@ os-api-ref>=1.0.0 # Apache-2.0
 oslosphinx>=4.7.0 # Apache-2.0
 oslotest>=1.10.0 # Apache-2.0
 os-testr>=0.8.0 # Apache-2.0
+osprofiler>=1.4.0 # Apache-2.0
 testrepository>=0.0.18 # Apache-2.0/BSD
 testresources>=0.2.4 # Apache-2.0/BSD
 testscenarios>=0.4 # Apache-2.0/BSD

tox.ini

@@ -9,6 +9,7 @@ usedevelop = True
 whitelist_externals = bash
                       find
                       rm
+                      env
 install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
 setenv = VIRTUAL_ENV={envdir}
          OS_TEST_PATH=./nova/tests/unit
@@ -28,16 +29,19 @@ passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY OS_DEB
 commands =
     {[testenv]commands}
     bash tools/pretty_tox.sh '{posargs}'
+    env TEST_OSPROFILER=1 bash tools/pretty_tox.sh 'nova.tests.unit.test_profiler'
 
 [testenv:py34]
 commands =
     {[testenv]commands}
     bash tools/pretty_tox3.sh '{posargs}'
+    env TEST_OSPROFILER=1 bash tools/pretty_tox.sh 'nova.tests.unit.test_profiler'
 
 [testenv:py35]
 commands =
     {[testenv]commands}
     bash tools/pretty_tox3.sh '{posargs}'
+    env TEST_OSPROFILER=1 bash tools/pretty_tox.sh 'nova.tests.unit.test_profiler'
 
 [testenv:pep8]
 basepython = python2.7