From 1291bb4736425b5b34fffb716532188f31286391 Mon Sep 17 00:00:00 2001 From: Ilya Shakhat Date: Wed, 29 Nov 2017 18:08:16 +0100 Subject: [PATCH] Add profiler support into Tempest The primary goal is to be able to run all Tempest tests and verify OpenStack when profiling is enabled. Also this patch allows to: * manually verify that certain services are properly instrumented and produce trace events when a scenario is executed; * write automatic tests for trace coverage; * profile certain tests from performance perspective. A new parameter is introduced into tempest.conf: * profiler.key - the key used to enable OSProfiler (should match the one configured in OpenStack services) To test the patch on DevStack: 1. Enable osprofiler with Redis collector in local.conf: enable_plugin osprofiler https://git.openstack.org/openstack/osprofiler master OSPROFILER_COLLECTOR=redis 2. Run all Tempest tests or select some, e.g.: tempest run --regex tempest.api.network.test_networks.NetworksTest.test_list_networks* Change-Id: I64f30c36adbf7fb26609142f22d3e305ac9e82b5 --- ...filer-config-options-db7c4ae6d338ee5c.yaml | 10 +++ tempest/config.py | 13 ++++ tempest/lib/common/profiler.py | 64 +++++++++++++++++++ tempest/lib/common/rest_client.py | 7 +- tempest/test.py | 6 ++ tempest/tests/lib/common/test_profiler.py | 63 ++++++++++++++++++ 6 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/add-profiler-config-options-db7c4ae6d338ee5c.yaml create mode 100644 tempest/lib/common/profiler.py create mode 100644 tempest/tests/lib/common/test_profiler.py diff --git a/releasenotes/notes/add-profiler-config-options-db7c4ae6d338ee5c.yaml b/releasenotes/notes/add-profiler-config-options-db7c4ae6d338ee5c.yaml new file mode 100644 index 0000000000..22450444df --- /dev/null +++ b/releasenotes/notes/add-profiler-config-options-db7c4ae6d338ee5c.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + Add support of `OSProfiler library`_ for profiling and distributed + tracing of OpenStack. A new config option ``key`` in section ``profiler`` + is added, the option sets the secret key used to enable profiling in + OpenStack services. The value needs to correspond to the one specified + in [profiler]/hmac_keys option of OpenStack services. + + .. _OSProfiler library: https://docs.openstack.org/osprofiler/ diff --git a/tempest/config.py b/tempest/config.py index fbe18a3e79..aeeed4706a 100644 --- a/tempest/config.py +++ b/tempest/config.py @@ -1100,6 +1100,18 @@ specify .* as the regex. """) ] + +profiler_group = cfg.OptGroup(name="profiler", + title="OpenStack Profiler") + +ProfilerGroup = [ + cfg.StrOpt('key', + help="The secret key to enable OpenStack Profiler. The value " + "should match the one configured in OpenStack services " + "under `[profiler]/hmac_keys` property. The default empty " + "value keeps profiling disabled"), +] + DefaultGroup = [ cfg.BoolOpt('pause_teardown', default=False, @@ -1132,6 +1144,7 @@ _opts = [ (service_available_group, ServiceAvailableGroup), (debug_group, DebugGroup), (placement_group, PlacementGroup), + (profiler_group, ProfilerGroup), (None, DefaultGroup) ] diff --git a/tempest/lib/common/profiler.py b/tempest/lib/common/profiler.py new file mode 100644 index 0000000000..15443371b3 --- /dev/null +++ b/tempest/lib/common/profiler.py @@ -0,0 +1,64 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import base64 +import hashlib +import hmac +import json + +from oslo_utils import encodeutils +from oslo_utils import uuidutils + +_profiler = {} + + +def enable(profiler_key, trace_id=None): + """Enable global profiler instance + + :param profiler_key: the secret key used to enable profiling in services + :param trace_id: unique id of the trace, if empty the id is generated + automatically + """ + _profiler['key'] = profiler_key + _profiler['uuid'] = trace_id or uuidutils.generate_uuid() + + +def disable(): + """Disable global profiler instance""" + _profiler.clear() + + +def serialize_as_http_headers(): + """Serialize profiler state as HTTP headers + + This function corresponds to the one from osprofiler library. + :return: dictionary with 2 keys `X-Trace-Info` and `X-Trace-HMAC`. + """ + p = _profiler + if not p: # profiler is not enabled + return {} + + info = {'base_id': p['uuid'], 'parent_id': p['uuid']} + trace_info = base64.urlsafe_b64encode( + encodeutils.to_utf8(json.dumps(info))) + trace_hmac = _sign(trace_info, p['key']) + + return { + 'X-Trace-Info': trace_info, + 'X-Trace-HMAC': trace_hmac, + } + + +def _sign(trace_info, key): + h = hmac.new(encodeutils.to_utf8(key), digestmod=hashlib.sha1) + h.update(trace_info) + return h.hexdigest() diff --git a/tempest/lib/common/rest_client.py b/tempest/lib/common/rest_client.py index e612bd1cca..6fd096de4d 100644 --- a/tempest/lib/common/rest_client.py +++ b/tempest/lib/common/rest_client.py @@ -27,6 +27,7 @@ from six.moves import urllib from tempest.lib.common import http from tempest.lib.common import jsonschema_validator +from tempest.lib.common import profiler from tempest.lib.common.utils import test_utils from tempest.lib import exceptions @@ -131,8 +132,10 @@ class RestClient(object): accept_type = 'json' if send_type is None: send_type = 'json' - return {'Content-Type': 'application/%s' % send_type, - 'Accept': 'application/%s' % accept_type} + headers = {'Content-Type': 'application/%s' % send_type, + 'Accept': 'application/%s' % accept_type} + headers.update(profiler.serialize_as_http_headers()) + return headers def __str__(self): STRING_LIMIT = 80 diff --git a/tempest/test.py b/tempest/test.py index c3c58dc51b..85000b6332 100644 --- a/tempest/test.py +++ b/tempest/test.py @@ -28,6 +28,7 @@ from tempest.common import credentials_factory as credentials from tempest.common import utils from tempest import config from tempest.lib.common import fixed_network +from tempest.lib.common import profiler from tempest.lib.common import validation_resources as vr from tempest.lib import decorators from tempest.lib import exceptions as lib_exc @@ -231,6 +232,9 @@ class BaseTestCase(testtools.testcase.WithAttributes, if CONF.pause_teardown: BaseTestCase.insert_pdb_breakpoint() + if CONF.profiler.key: + profiler.disable() + @classmethod def insert_pdb_breakpoint(cls): """Add pdb breakpoint. @@ -608,6 +612,8 @@ class BaseTestCase(testtools.testcase.WithAttributes, self.useFixture(fixtures.LoggerFixture(nuke_handlers=False, format=self.log_format, level=None)) + if CONF.profiler.key: + profiler.enable(CONF.profiler.key) @property def credentials_provider(self): diff --git a/tempest/tests/lib/common/test_profiler.py b/tempest/tests/lib/common/test_profiler.py new file mode 100644 index 0000000000..59fa036445 --- /dev/null +++ b/tempest/tests/lib/common/test_profiler.py @@ -0,0 +1,63 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import testtools + +from tempest.lib.common import profiler + + +class TestProfiler(testtools.TestCase): + + def test_serialize(self): + key = 'SECRET_KEY' + pm = {'key': key, 'uuid': 'ID'} + + with mock.patch('tempest.lib.common.profiler._profiler', pm): + with mock.patch('json.dumps') as jdm: + jdm.return_value = '{"base_id": "ID", "parent_id": "ID"}' + + expected = { + 'X-Trace-HMAC': + '887292df9f13b8b5ecd6bbbd2e16bfaaa4d914b0', + 'X-Trace-Info': + b'eyJiYXNlX2lkIjogIklEIiwgInBhcmVudF9pZCI6ICJJRCJ9' + } + + self.assertEqual(expected, + profiler.serialize_as_http_headers()) + + def test_profiler_lifecycle(self): + key = 'SECRET_KEY' + uuid = 'ID' + + self.assertEqual({}, profiler._profiler) + + profiler.enable(key, uuid) + self.assertEqual({'key': key, 'uuid': uuid}, profiler._profiler) + + profiler.disable() + self.assertEqual({}, profiler._profiler) + + @mock.patch('oslo_utils.uuidutils.generate_uuid') + def test_profiler_lifecycle_generate_trace_id(self, generate_uuid_mock): + key = 'SECRET_KEY' + uuid = 'ID' + generate_uuid_mock.return_value = uuid + + self.assertEqual({}, profiler._profiler) + + profiler.enable(key) + self.assertEqual({'key': key, 'uuid': uuid}, profiler._profiler) + + profiler.disable() + self.assertEqual({}, profiler._profiler)