Add profiling support to novaclient

To be able to create profiling traces for Nova, client should be
able to send special HTTP header that contains trace info.
This patch is also important to be able to make cross project
traces. (Typical case heat calls nova via python client, if
profiler is initialized in heat, nova client will add extra
header, that will be parsed by special osprofiler middleware in nova
api.)

Security considerations: trace information is signed by one of the
HMAC keys that are set in nova.conf. So only person who knows HMAC key
is able to send proper header.

oslo-spec: https://review.openstack.org/#/c/103825/
Based on: https://review.openstack.org/#/c/105089/

Co-Authored-By: Dina Belova <dbelova@mirantis.com>
Co-Authored-By: Roman Podoliaka <rpodolyaka@mirantis.com>
Co-Authored-By: Tovin Seven <vinhnt@vn.fujitsu.com>

Partially implements: blueprint osprofiler-support-in-nova

Depends-On: I82d2badc8c1fcec27c3fce7c3c20e0f3b76414f1
Change-Id: I56ce4b547230e475854994c9d2249ef90e5b656c
This commit is contained in:
Boris Pavlovic 2015-12-08 14:17:45 +03:00 committed by Tovin Seven
parent c732a5edce
commit 0fed79fd8f
6 changed files with 101 additions and 5 deletions

View File

@ -30,6 +30,9 @@ from keystoneauth1 import session as ksession
from oslo_utils import importutils
import pkg_resources
osprofiler_profiler = importutils.try_import("osprofiler.profiler")
osprofiler_web = importutils.try_import("osprofiler.web")
from novaclient import api_versions
from novaclient import exceptions
from novaclient import extension as ext
@ -67,6 +70,12 @@ class SessionClient(adapter.LegacyJsonAdapter):
def request(self, url, method, **kwargs):
kwargs.setdefault('headers', kwargs.get('headers', {}))
api_versions.update_headers(kwargs["headers"], self.api_version)
# NOTE(dbelova): osprofiler_web.get_trace_id_headers does not add any
# headers in case if osprofiler is not initialized.
if osprofiler_web:
kwargs['headers'].update(osprofiler_web.get_trace_id_headers())
# NOTE(jamielennox): The standard call raises errors from
# keystoneauth1, where we need to raise the novaclient errors.
raise_exc = kwargs.pop('raise_exc', True)
@ -343,6 +352,17 @@ def Client(version, username=None, password=None, project_id=None,
api_version, client_class = _get_client_class_and_version(version)
kwargs.pop("direct_use", None)
profile = kwargs.pop("profile", None)
if osprofiler_profiler and profile:
# Initialize the root of the future trace: the created trace ID will
# be used as the very first parent to which all related traces will be
# bound to. The given HMAC key must correspond to the one set in
# nova-api nova.conf, otherwise the latter will fail to check the
# request signature and will skip initialization of osprofiler on
# the server side.
osprofiler_profiler.init(profile)
return client_class(api_version=api_version,
auth_url=auth_url,
direct_use=False,

View File

@ -30,6 +30,8 @@ from oslo_utils import importutils
from oslo_utils import strutils
import six
osprofiler_profiler = importutils.try_import("osprofiler.profiler")
HAS_KEYRING = False
all_errors = ValueError
try:
@ -509,6 +511,19 @@ class OpenStackComputeShell(object):
dest='endpoint_override',
help=argparse.SUPPRESS)
if osprofiler_profiler:
parser.add_argument('--profile',
metavar='HMAC_KEY',
help='HMAC key to use for encrypting context '
'data for performance profiling of operation. '
'This key should be the value of the HMAC key '
'configured for the OSprofiler middleware in '
'nova; it is specified in the Nova '
'configuration file at "/etc/nova/nova.conf". '
'Without the key, profiling will not be '
'triggered even if OSprofiler is enabled on '
'the server side.')
self._append_global_identity_args(parser, argv)
return parser
@ -749,6 +764,10 @@ class OpenStackComputeShell(object):
_("You must provide an auth url "
"via either --os-auth-url or env[OS_AUTH_URL]"))
additional_kwargs = {}
if osprofiler_profiler:
additional_kwargs["profile"] = args.profile
# This client is just used to discover api version. Version API needn't
# microversion, so we just pass version 2 at here.
self.cs = client.Client(
@ -767,7 +786,8 @@ class OpenStackComputeShell(object):
project_domain_id=os_project_domain_id,
project_domain_name=os_project_domain_name,
user_domain_id=os_user_domain_id,
user_domain_name=os_user_domain_name)
user_domain_name=os_user_domain_name,
**additional_kwargs)
if not skip_auth:
if not api_version.is_latest():
@ -858,6 +878,11 @@ class OpenStackComputeShell(object):
args.func(self.cs, args)
if osprofiler_profiler and args.profile:
trace_id = osprofiler_profiler.get().get_base_id()
print("To display trace use the command:\n\n"
" osprofiler trace show --html %s " % trace_id)
if args.timings:
self._dump_timings(self.times + self.cs.get_timings())

View File

@ -24,7 +24,8 @@ COMPUTE_URL = 'http://compute.host'
class V1(fixtures.Fixture):
def __init__(self, requests_mock,
compute_url=COMPUTE_URL, identity_url=IDENTITY_URL):
compute_url=COMPUTE_URL, identity_url=IDENTITY_URL,
**client_kwargs):
super(V1, self).__init__()
self.identity_url = identity_url
self.compute_url = compute_url
@ -41,6 +42,8 @@ class V1(fixtures.Fixture):
s = self.token.add_service('computev3')
s.add_endpoint(self.compute_url)
self._client_kwargs = client_kwargs
def setUp(self):
super(V1, self).setUp()
@ -52,13 +55,14 @@ class V1(fixtures.Fixture):
self.requests_mock.get(self.identity_url,
json=self.discovery,
headers=headers)
self.client = self.new_client()
self.client = self.new_client(**self._client_kwargs)
def new_client(self):
def new_client(self, **client_kwargs):
return client.Client("2", username='xx',
password='xx',
project_id='xx',
auth_url=self.identity_url)
auth_url=self.identity_url,
**client_kwargs)
class SessionV1(V1):

View File

@ -655,6 +655,27 @@ class ShellTest(utils.TestCase):
exc = self.assertRaises(RuntimeError, self.shell, '--timings list')
self.assertEqual('Boom!', str(exc))
@requests_mock.Mocker()
def test_osprofiler(self, m_requests):
self.make_env()
def client(*args, **kwargs):
self.assertEqual('swordfish', kwargs['profile'])
with mock.patch('novaclient.client.Client', client):
# we are only interested in the fact Client is initialized properly
self.shell('list --profile swordfish', (0, 2))
@requests_mock.Mocker()
def test_osprofiler_not_installed(self, m_requests):
self.make_env()
# NOTE(rpodolyaka): osprofiler is in test-requirements, so we have to
# simulate its absence here
with mock.patch('novaclient.shell.osprofiler_profiler', None):
_, stderr = self.shell('list --profile swordfish', (0, 2))
self.assertIn('unrecognized arguments: --profile swordfish',
stderr)
@mock.patch('novaclient.shell.SecretsHelper.tenant_id',
return_value=True)
@mock.patch('novaclient.shell.SecretsHelper.auth_token',

View File

@ -0,0 +1,25 @@
---
prelude: >
OSprofiler support was added to the client. That makes
possible to trigger Nova operation trace generation from
the CLI.
features:
- A new ``--profile`` option was added to allow Nova
profiling from the CLI. If the user wishes to trace a
nova boot request he or she needs to type the following
command -- ``nova --profile <secret_key> boot --image <image>
--flavor <flavor> <vm_name>``, where ``secret_key`` should match one
of the keys defined in nova.conf. As a result of this operation
additional information regarding ``trace_id`` will be
printed, that can be used to generate human-friendly
html report -- ``osprofiler trace show --html <trace_id> --out
trace.html``.
To enable profiling, user needs to have osprofiler
installed in the local environment via ``pip install osprofiler``.
security:
- OSprofiler support, that was added during the Ocata release cycle,
requires passing of trace information between various
OpenStack services. This information is signed by one of
the HMAC keys defined in nova.conf file. That means that
only someone who knows this key is able to send the proper
header to trigger profiling.

View File

@ -15,6 +15,7 @@ requests-mock>=1.1 # Apache-2.0
sphinx!=1.3b1,<1.4,>=1.2.1 # BSD
os-client-config>=1.22.0 # Apache-2.0
oslosphinx>=4.7.0 # Apache-2.0
osprofiler>=1.4.0 # Apache-2.0
testrepository>=0.0.18 # Apache-2.0/BSD
testscenarios>=0.4 # Apache-2.0/BSD
testtools>=1.4.0 # MIT