Add profiling support to novaclient

To be able to create profiling traces for Nova, client should be
able to send special HTTP header that contains trace info.
This patch is also important to be able to make cross project
traces. (Typical case heat calls nova via python client, if
profiler is initialized in heat, nova client will add extra
header, that will be parsed by special osprofiler middleware in nova
api.)

Security considerations: trace information is signed by one of the
HMAC keys that are set in nova.conf. So only person who knows HMAC key
is able to send proper header.

oslo-spec: https://review.openstack.org/#/c/103825/
Based on: https://review.openstack.org/#/c/105089/

Co-Authored-By: Dina Belova <dbelova@mirantis.com>
Co-Authored-By: Roman Podoliaka <rpodolyaka@mirantis.com>
Co-Authored-By: Tovin Seven <vinhnt@vn.fujitsu.com>

Partially implements: blueprint osprofiler-support-in-nova

Depends-On: I82d2badc8c1fcec27c3fce7c3c20e0f3b76414f1
Change-Id: I56ce4b547230e475854994c9d2249ef90e5b656c

novaclient/client.py

@@ -30,6 +30,9 @@ from keystoneauth1 import session as ksession
 from oslo_utils import importutils
 import pkg_resources
 
+osprofiler_profiler = importutils.try_import("osprofiler.profiler")
+osprofiler_web = importutils.try_import("osprofiler.web")
+
 from novaclient import api_versions
 from novaclient import exceptions
 from novaclient import extension as ext
@@ -67,6 +70,12 @@ class SessionClient(adapter.LegacyJsonAdapter):
     def request(self, url, method, **kwargs):
         kwargs.setdefault('headers', kwargs.get('headers', {}))
         api_versions.update_headers(kwargs["headers"], self.api_version)
+
+        # NOTE(dbelova): osprofiler_web.get_trace_id_headers does not add any
+        # headers in case if osprofiler is not initialized.
+        if osprofiler_web:
+            kwargs['headers'].update(osprofiler_web.get_trace_id_headers())
+
         # NOTE(jamielennox): The standard call raises errors from
         # keystoneauth1, where we need to raise the novaclient errors.
         raise_exc = kwargs.pop('raise_exc', True)
@@ -343,6 +352,17 @@ def Client(version, username=None, password=None, project_id=None,
 
     api_version, client_class = _get_client_class_and_version(version)
     kwargs.pop("direct_use", None)
+
+    profile = kwargs.pop("profile", None)
+    if osprofiler_profiler and profile:
+        # Initialize the root of the future trace: the created trace ID will
+        # be used as the very first parent to which all related traces will be
+        # bound to. The given HMAC key must correspond to the one set in
+        # nova-api nova.conf, otherwise the latter will fail to check the
+        # request signature and will skip initialization of osprofiler on
+        # the server side.
+        osprofiler_profiler.init(profile)
+
     return client_class(api_version=api_version,
                         auth_url=auth_url,
                         direct_use=False,

novaclient/shell.py

@@ -30,6 +30,8 @@ from oslo_utils import importutils
 from oslo_utils import strutils
 import six
 
+osprofiler_profiler = importutils.try_import("osprofiler.profiler")
+
 HAS_KEYRING = False
 all_errors = ValueError
 try:
@@ -509,6 +511,19 @@ class OpenStackComputeShell(object):
             dest='endpoint_override',
             help=argparse.SUPPRESS)
 
+        if osprofiler_profiler:
+            parser.add_argument('--profile',
+                                metavar='HMAC_KEY',
+                                help='HMAC key to use for encrypting context '
+                                'data for performance profiling of operation. '
+                                'This key should be the value of the HMAC key '
+                                'configured for the OSprofiler middleware in '
+                                'nova; it is specified in the Nova '
+                                'configuration file at "/etc/nova/nova.conf". '
+                                'Without the key, profiling will not be '
+                                'triggered even if OSprofiler is enabled on '
+                                'the server side.')
+
         self._append_global_identity_args(parser, argv)
 
         return parser
@@ -749,6 +764,10 @@ class OpenStackComputeShell(object):
                 _("You must provide an auth url "
                   "via either --os-auth-url or env[OS_AUTH_URL]"))
 
+        additional_kwargs = {}
+        if osprofiler_profiler:
+            additional_kwargs["profile"] = args.profile
+
         # This client is just used to discover api version. Version API needn't
         # microversion, so we just pass version 2 at here.
         self.cs = client.Client(
@@ -767,7 +786,8 @@ class OpenStackComputeShell(object):
             project_domain_id=os_project_domain_id,
             project_domain_name=os_project_domain_name,
             user_domain_id=os_user_domain_id,
-            user_domain_name=os_user_domain_name)
+            user_domain_name=os_user_domain_name,
+            **additional_kwargs)
 
         if not skip_auth:
             if not api_version.is_latest():
@@ -858,6 +878,11 @@ class OpenStackComputeShell(object):
 
         args.func(self.cs, args)
 
+        if osprofiler_profiler and args.profile:
+            trace_id = osprofiler_profiler.get().get_base_id()
+            print("To display trace use the command:\n\n"
+                  "  osprofiler trace show --html %s " % trace_id)
+
         if args.timings:
             self._dump_timings(self.times + self.cs.get_timings())
 

novaclient/tests/unit/fixture_data/client.py

@@ -24,7 +24,8 @@ COMPUTE_URL = 'http://compute.host'
 class V1(fixtures.Fixture):
 
     def __init__(self, requests_mock,
-                 compute_url=COMPUTE_URL, identity_url=IDENTITY_URL):
+                 compute_url=COMPUTE_URL, identity_url=IDENTITY_URL,
+                 **client_kwargs):
         super(V1, self).__init__()
         self.identity_url = identity_url
         self.compute_url = compute_url
@@ -41,6 +42,8 @@ class V1(fixtures.Fixture):
         s = self.token.add_service('computev3')
         s.add_endpoint(self.compute_url)
 
+        self._client_kwargs = client_kwargs
+
     def setUp(self):
         super(V1, self).setUp()
 
@@ -52,13 +55,14 @@ class V1(fixtures.Fixture):
         self.requests_mock.get(self.identity_url,
                                json=self.discovery,
                                headers=headers)
-        self.client = self.new_client()
+        self.client = self.new_client(**self._client_kwargs)
 
-    def new_client(self):
+    def new_client(self, **client_kwargs):
         return client.Client("2", username='xx',
                              password='xx',
                              project_id='xx',
-                             auth_url=self.identity_url)
+                             auth_url=self.identity_url,
+                             **client_kwargs)
 
 
 class SessionV1(V1):

novaclient/tests/unit/test_shell.py

@@ -655,6 +655,27 @@ class ShellTest(utils.TestCase):
         exc = self.assertRaises(RuntimeError, self.shell, '--timings list')
         self.assertEqual('Boom!', str(exc))
 
+    @requests_mock.Mocker()
+    def test_osprofiler(self, m_requests):
+        self.make_env()
+
+        def client(*args, **kwargs):
+            self.assertEqual('swordfish', kwargs['profile'])
+        with mock.patch('novaclient.client.Client', client):
+            # we are only interested in the fact Client is initialized properly
+            self.shell('list --profile swordfish', (0, 2))
+
+    @requests_mock.Mocker()
+    def test_osprofiler_not_installed(self, m_requests):
+        self.make_env()
+
+        # NOTE(rpodolyaka): osprofiler is in test-requirements, so we have to
+        # simulate its absence here
+        with mock.patch('novaclient.shell.osprofiler_profiler', None):
+            _, stderr = self.shell('list --profile swordfish', (0, 2))
+            self.assertIn('unrecognized arguments: --profile swordfish',
+                          stderr)
+
     @mock.patch('novaclient.shell.SecretsHelper.tenant_id',
                 return_value=True)
     @mock.patch('novaclient.shell.SecretsHelper.auth_token',

releasenotes/notes/add-osprofiler-support-cc9dd228242e9919.yaml

@@ -0,0 +1,25 @@
+---
+prelude: >
+    OSprofiler support was added to the client. That makes
+    possible to trigger Nova operation trace generation from
+    the CLI.
+features:
+  - A new ``--profile`` option was added to allow Nova
+    profiling from the CLI. If the user wishes to trace a
+    nova boot request he or she needs to type the following
+    command -- ``nova --profile <secret_key> boot --image <image>
+    --flavor <flavor> <vm_name>``, where ``secret_key`` should match one
+    of the keys defined in nova.conf. As a result of this operation
+    additional information regarding ``trace_id`` will be
+    printed, that can be used to generate human-friendly
+    html report -- ``osprofiler trace show --html <trace_id> --out
+    trace.html``.
+    To enable profiling, user needs to have osprofiler
+    installed in the local environment via ``pip install osprofiler``.
+security:
+  - OSprofiler support, that was added during the Ocata release cycle,
+    requires passing of trace information between various
+    OpenStack services. This information is signed by one of
+    the HMAC keys defined in nova.conf file. That means that
+    only someone who knows this key is able to send the proper
+    header to trigger profiling.

test-requirements.txt

@@ -15,6 +15,7 @@ requests-mock>=1.1 # Apache-2.0
 sphinx!=1.3b1,<1.4,>=1.2.1 # BSD
 os-client-config>=1.22.0 # Apache-2.0
 oslosphinx>=4.7.0 # Apache-2.0
+osprofiler>=1.4.0 # Apache-2.0
 testrepository>=0.0.18 # Apache-2.0/BSD
 testscenarios>=0.4 # Apache-2.0/BSD
 testtools>=1.4.0 # MIT