Add profiler support into Tempest

The primary goal is to be able to run all Tempest tests
and verify OpenStack when profiling is enabled. Also this patch
allows to:
 * manually verify that certain services are properly instrumented
   and produce trace events when a scenario is executed;
 * write automatic tests for trace coverage;
 * profile certain tests from performance perspective.

A new parameter is introduced into tempest.conf:
 * profiler.key - the key used to enable OSProfiler (should
     match the one configured in OpenStack services)

To test the patch on DevStack:
 1. Enable osprofiler with Redis collector in local.conf:

   enable_plugin osprofiler https://git.openstack.org/openstack/osprofiler master
   OSPROFILER_COLLECTOR=redis

 2. Run all Tempest tests or select some, e.g.:

   tempest run --regex tempest.api.network.test_networks.NetworksTest.test_list_networks*

Change-Id: I64f30c36adbf7fb26609142f22d3e305ac9e82b5
This commit is contained in:
Ilya Shakhat 2017-11-29 18:08:16 +01:00 committed by Tovin Seven
parent 4b8a7b8638
commit 1291bb4736
6 changed files with 161 additions and 2 deletions

View File

@ -0,0 +1,10 @@
---
features:
- |
Add support of `OSProfiler library`_ for profiling and distributed
tracing of OpenStack. A new config option ``key`` in section ``profiler``
is added, the option sets the secret key used to enable profiling in
OpenStack services. The value needs to correspond to the one specified
in [profiler]/hmac_keys option of OpenStack services.
.. _OSProfiler library: https://docs.openstack.org/osprofiler/

View File

@ -1100,6 +1100,18 @@ specify .* as the regex.
""")
]
profiler_group = cfg.OptGroup(name="profiler",
title="OpenStack Profiler")
ProfilerGroup = [
cfg.StrOpt('key',
help="The secret key to enable OpenStack Profiler. The value "
"should match the one configured in OpenStack services "
"under `[profiler]/hmac_keys` property. The default empty "
"value keeps profiling disabled"),
]
DefaultGroup = [
cfg.BoolOpt('pause_teardown',
default=False,
@ -1132,6 +1144,7 @@ _opts = [
(service_available_group, ServiceAvailableGroup),
(debug_group, DebugGroup),
(placement_group, PlacementGroup),
(profiler_group, ProfilerGroup),
(None, DefaultGroup)
]

View File

@ -0,0 +1,64 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import base64
import hashlib
import hmac
import json
from oslo_utils import encodeutils
from oslo_utils import uuidutils
_profiler = {}
def enable(profiler_key, trace_id=None):
"""Enable global profiler instance
:param profiler_key: the secret key used to enable profiling in services
:param trace_id: unique id of the trace, if empty the id is generated
automatically
"""
_profiler['key'] = profiler_key
_profiler['uuid'] = trace_id or uuidutils.generate_uuid()
def disable():
"""Disable global profiler instance"""
_profiler.clear()
def serialize_as_http_headers():
"""Serialize profiler state as HTTP headers
This function corresponds to the one from osprofiler library.
:return: dictionary with 2 keys `X-Trace-Info` and `X-Trace-HMAC`.
"""
p = _profiler
if not p: # profiler is not enabled
return {}
info = {'base_id': p['uuid'], 'parent_id': p['uuid']}
trace_info = base64.urlsafe_b64encode(
encodeutils.to_utf8(json.dumps(info)))
trace_hmac = _sign(trace_info, p['key'])
return {
'X-Trace-Info': trace_info,
'X-Trace-HMAC': trace_hmac,
}
def _sign(trace_info, key):
h = hmac.new(encodeutils.to_utf8(key), digestmod=hashlib.sha1)
h.update(trace_info)
return h.hexdigest()

View File

@ -27,6 +27,7 @@ from six.moves import urllib
from tempest.lib.common import http
from tempest.lib.common import jsonschema_validator
from tempest.lib.common import profiler
from tempest.lib.common.utils import test_utils
from tempest.lib import exceptions
@ -131,8 +132,10 @@ class RestClient(object):
accept_type = 'json'
if send_type is None:
send_type = 'json'
return {'Content-Type': 'application/%s' % send_type,
headers = {'Content-Type': 'application/%s' % send_type,
'Accept': 'application/%s' % accept_type}
headers.update(profiler.serialize_as_http_headers())
return headers
def __str__(self):
STRING_LIMIT = 80

View File

@ -28,6 +28,7 @@ from tempest.common import credentials_factory as credentials
from tempest.common import utils
from tempest import config
from tempest.lib.common import fixed_network
from tempest.lib.common import profiler
from tempest.lib.common import validation_resources as vr
from tempest.lib import decorators
from tempest.lib import exceptions as lib_exc
@ -231,6 +232,9 @@ class BaseTestCase(testtools.testcase.WithAttributes,
if CONF.pause_teardown:
BaseTestCase.insert_pdb_breakpoint()
if CONF.profiler.key:
profiler.disable()
@classmethod
def insert_pdb_breakpoint(cls):
"""Add pdb breakpoint.
@ -608,6 +612,8 @@ class BaseTestCase(testtools.testcase.WithAttributes,
self.useFixture(fixtures.LoggerFixture(nuke_handlers=False,
format=self.log_format,
level=None))
if CONF.profiler.key:
profiler.enable(CONF.profiler.key)
@property
def credentials_provider(self):

View File

@ -0,0 +1,63 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
import testtools
from tempest.lib.common import profiler
class TestProfiler(testtools.TestCase):
def test_serialize(self):
key = 'SECRET_KEY'
pm = {'key': key, 'uuid': 'ID'}
with mock.patch('tempest.lib.common.profiler._profiler', pm):
with mock.patch('json.dumps') as jdm:
jdm.return_value = '{"base_id": "ID", "parent_id": "ID"}'
expected = {
'X-Trace-HMAC':
'887292df9f13b8b5ecd6bbbd2e16bfaaa4d914b0',
'X-Trace-Info':
b'eyJiYXNlX2lkIjogIklEIiwgInBhcmVudF9pZCI6ICJJRCJ9'
}
self.assertEqual(expected,
profiler.serialize_as_http_headers())
def test_profiler_lifecycle(self):
key = 'SECRET_KEY'
uuid = 'ID'
self.assertEqual({}, profiler._profiler)
profiler.enable(key, uuid)
self.assertEqual({'key': key, 'uuid': uuid}, profiler._profiler)
profiler.disable()
self.assertEqual({}, profiler._profiler)
@mock.patch('oslo_utils.uuidutils.generate_uuid')
def test_profiler_lifecycle_generate_trace_id(self, generate_uuid_mock):
key = 'SECRET_KEY'
uuid = 'ID'
generate_uuid_mock.return_value = uuid
self.assertEqual({}, profiler._profiler)
profiler.enable(key)
self.assertEqual({'key': key, 'uuid': uuid}, profiler._profiler)
profiler.disable()
self.assertEqual({}, profiler._profiler)