Add profiler support into Tempest

The primary goal is to be able to run all Tempest tests
and verify OpenStack when profiling is enabled. Also this patch
allows to:
 * manually verify that certain services are properly instrumented
   and produce trace events when a scenario is executed;
 * write automatic tests for trace coverage;
 * profile certain tests from performance perspective.

A new parameter is introduced into tempest.conf:
 * profiler.key - the key used to enable OSProfiler (should
     match the one configured in OpenStack services)

To test the patch on DevStack:
 1. Enable osprofiler with Redis collector in local.conf:

   enable_plugin osprofiler https://git.openstack.org/openstack/osprofiler master
   OSPROFILER_COLLECTOR=redis

 2. Run all Tempest tests or select some, e.g.:

   tempest run --regex tempest.api.network.test_networks.NetworksTest.test_list_networks*

Change-Id: I64f30c36adbf7fb26609142f22d3e305ac9e82b5

releasenotes/notes/add-profiler-config-options-db7c4ae6d338ee5c.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    Add support of `OSProfiler library`_ for profiling and distributed
+    tracing of OpenStack. A new config option ``key`` in section ``profiler``
+    is added, the option sets the secret key used to enable profiling in
+    OpenStack services. The value needs to correspond to the one specified
+    in [profiler]/hmac_keys option of OpenStack services.
+
+    .. _OSProfiler library: https://docs.openstack.org/osprofiler/

tempest/config.py

@@ -1100,6 +1100,18 @@ specify .* as the regex.
 """)
 ]
 
+
+profiler_group = cfg.OptGroup(name="profiler",
+                              title="OpenStack Profiler")
+
+ProfilerGroup = [
+    cfg.StrOpt('key',
+               help="The secret key to enable OpenStack Profiler. The value "
+                    "should match the one configured in OpenStack services "
+                    "under `[profiler]/hmac_keys` property. The default empty "
+                    "value keeps profiling disabled"),
+]
+
 DefaultGroup = [
     cfg.BoolOpt('pause_teardown',
                 default=False,
@@ -1132,6 +1144,7 @@ _opts = [
     (service_available_group, ServiceAvailableGroup),
     (debug_group, DebugGroup),
     (placement_group, PlacementGroup),
+    (profiler_group, ProfilerGroup),
     (None, DefaultGroup)
 ]
 

tempest/lib/common/profiler.py

@@ -0,0 +1,64 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import base64
+import hashlib
+import hmac
+import json
+
+from oslo_utils import encodeutils
+from oslo_utils import uuidutils
+
+_profiler = {}
+
+
+def enable(profiler_key, trace_id=None):
+    """Enable global profiler instance
+
+    :param profiler_key: the secret key used to enable profiling in services
+    :param trace_id: unique id of the trace, if empty the id is generated
+        automatically
+    """
+    _profiler['key'] = profiler_key
+    _profiler['uuid'] = trace_id or uuidutils.generate_uuid()
+
+
+def disable():
+    """Disable global profiler instance"""
+    _profiler.clear()
+
+
+def serialize_as_http_headers():
+    """Serialize profiler state as HTTP headers
+
+    This function corresponds to the one from osprofiler library.
+    :return: dictionary with 2 keys `X-Trace-Info` and `X-Trace-HMAC`.
+    """
+    p = _profiler
+    if not p:  # profiler is not enabled
+        return {}
+
+    info = {'base_id': p['uuid'], 'parent_id': p['uuid']}
+    trace_info = base64.urlsafe_b64encode(
+        encodeutils.to_utf8(json.dumps(info)))
+    trace_hmac = _sign(trace_info, p['key'])
+
+    return {
+        'X-Trace-Info': trace_info,
+        'X-Trace-HMAC': trace_hmac,
+    }
+
+
+def _sign(trace_info, key):
+    h = hmac.new(encodeutils.to_utf8(key), digestmod=hashlib.sha1)
+    h.update(trace_info)
+    return h.hexdigest()

tempest/lib/common/rest_client.py

@@ -27,6 +27,7 @@ from six.moves import urllib
 
 from tempest.lib.common import http
 from tempest.lib.common import jsonschema_validator
+from tempest.lib.common import profiler
 from tempest.lib.common.utils import test_utils
 from tempest.lib import exceptions
 
@@ -131,8 +132,10 @@ class RestClient(object):
             accept_type = 'json'
         if send_type is None:
             send_type = 'json'
-        return {'Content-Type': 'application/%s' % send_type,
+        headers = {'Content-Type': 'application/%s' % send_type,
                    'Accept': 'application/%s' % accept_type}
+        headers.update(profiler.serialize_as_http_headers())
+        return headers
 
     def __str__(self):
         STRING_LIMIT = 80

tempest/test.py

@@ -28,6 +28,7 @@ from tempest.common import credentials_factory as credentials
 from tempest.common import utils
 from tempest import config
 from tempest.lib.common import fixed_network
+from tempest.lib.common import profiler
 from tempest.lib.common import validation_resources as vr
 from tempest.lib import decorators
 from tempest.lib import exceptions as lib_exc
@@ -231,6 +232,9 @@ class BaseTestCase(testtools.testcase.WithAttributes,
         if CONF.pause_teardown:
             BaseTestCase.insert_pdb_breakpoint()
 
+        if CONF.profiler.key:
+            profiler.disable()
+
     @classmethod
     def insert_pdb_breakpoint(cls):
         """Add pdb breakpoint.
@@ -608,6 +612,8 @@ class BaseTestCase(testtools.testcase.WithAttributes,
             self.useFixture(fixtures.LoggerFixture(nuke_handlers=False,
                                                    format=self.log_format,
                                                    level=None))
+        if CONF.profiler.key:
+            profiler.enable(CONF.profiler.key)
 
     @property
     def credentials_provider(self):

tempest/tests/lib/common/test_profiler.py

@@ -0,0 +1,63 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+import testtools
+
+from tempest.lib.common import profiler
+
+
+class TestProfiler(testtools.TestCase):
+
+    def test_serialize(self):
+        key = 'SECRET_KEY'
+        pm = {'key': key, 'uuid': 'ID'}
+
+        with mock.patch('tempest.lib.common.profiler._profiler', pm):
+            with mock.patch('json.dumps') as jdm:
+                jdm.return_value = '{"base_id": "ID", "parent_id": "ID"}'
+
+                expected = {
+                    'X-Trace-HMAC':
+                        '887292df9f13b8b5ecd6bbbd2e16bfaaa4d914b0',
+                    'X-Trace-Info':
+                        b'eyJiYXNlX2lkIjogIklEIiwgInBhcmVudF9pZCI6ICJJRCJ9'
+                }
+
+                self.assertEqual(expected,
+                                 profiler.serialize_as_http_headers())
+
+    def test_profiler_lifecycle(self):
+        key = 'SECRET_KEY'
+        uuid = 'ID'
+
+        self.assertEqual({}, profiler._profiler)
+
+        profiler.enable(key, uuid)
+        self.assertEqual({'key': key, 'uuid': uuid}, profiler._profiler)
+
+        profiler.disable()
+        self.assertEqual({}, profiler._profiler)
+
+    @mock.patch('oslo_utils.uuidutils.generate_uuid')
+    def test_profiler_lifecycle_generate_trace_id(self, generate_uuid_mock):
+        key = 'SECRET_KEY'
+        uuid = 'ID'
+        generate_uuid_mock.return_value = uuid
+
+        self.assertEqual({}, profiler._profiler)
+
+        profiler.enable(key)
+        self.assertEqual({'key': key, 'uuid': uuid}, profiler._profiler)
+
+        profiler.disable()
+        self.assertEqual({}, profiler._profiler)