metadata: use requests for comms with nova api

httplib2 makes use of the ssl module provided by Python; under Python 2,
the ssl module does not support IP addresses as subject alternate names
(SAN's) which although an optional part of the associated RFC, is awkward
to work with in environments where certificate management approaches
rely on use of IP addresses in SAN's.

The requests module is more than happy to deal with this scenario; switch
to requests in preference of httplib2 for metadata proxy calls.

httplib2 is retained as its used elsewhere in the codebase.

Closes-Bug: 1790598
Change-Id: Ife4adf09ddbf7116da2f8596c80aed53fb6790df
(cherry picked from commit 7e0dd2f18d)

neutron/agent/metadata/agent.py

@@ -15,7 +15,6 @@
 import hashlib
 import hmac
 
-import httplib2
 from neutron_lib.agent import topics
 from neutron_lib import constants
 from neutron_lib import context
@@ -24,6 +23,7 @@ from oslo_log import log as logging
 import oslo_messaging
 from oslo_service import loopingcall
 from oslo_utils import encodeutils
+import requests
 import six
 import six.moves.urllib.parse as urlparse
 import webob
@@ -186,35 +186,41 @@ class MetadataProxyHandler(object):
             req.query_string,
             ''))
 
-        h = httplib2.Http(
+        disable_ssl_certificate_validation = self.conf.nova_metadata_insecure
-            ca_certs=self.conf.auth_ca_cert,
+        if self.conf.auth_ca_cert and not disable_ssl_certificate_validation:
-            disable_ssl_certificate_validation=self.conf.nova_metadata_insecure
+            verify_cert = self.conf.auth_ca_cert
-        )
+        else:
-        if self.conf.nova_client_cert and self.conf.nova_client_priv_key:
+            verify_cert = not disable_ssl_certificate_validation
-            h.add_certificate(self.conf.nova_client_priv_key,
-                              self.conf.nova_client_cert,
-                              nova_host_port)
-        resp, content = h.request(url, method=req.method, headers=headers,
-                                  body=req.body)
 
-        if resp.status == 200:
+        client_cert = None
-            req.response.content_type = resp['content-type']
+        if self.conf.nova_client_cert and self.conf.nova_client_priv_key:
-            req.response.body = content
+            client_cert = (self.conf.nova_client_cert,
+                           self.conf.nova_client_priv_key)
+
+        resp = requests.request(method=req.method, url=url,
+                                headers=headers,
+                                data=req.body,
+                                cert=client_cert,
+                                verify=verify_cert)
+
+        if resp.status_code == 200:
+            req.response.content_type = resp.headers['content-type']
+            req.response.body = resp.content
             LOG.debug(str(resp))
             return req.response
-        elif resp.status == 403:
+        elif resp.status_code == 403:
             LOG.warning(
                 'The remote metadata server responded with Forbidden. This '
                 'response usually occurs when shared secrets do not match.'
             )
             return webob.exc.HTTPForbidden()
-        elif resp.status == 400:
+        elif resp.status_code == 400:
             return webob.exc.HTTPBadRequest()
-        elif resp.status == 404:
+        elif resp.status_code == 404:
             return webob.exc.HTTPNotFound()
-        elif resp.status == 409:
+        elif resp.status_code == 409:
             return webob.exc.HTTPConflict()
-        elif resp.status == 500:
+        elif resp.status_code == 500:
             msg = _(
                 'Remote metadata server experienced an internal server error.'
             )

neutron/tests/unit/agent/metadata/test_agent.py

@@ -339,37 +339,28 @@ class _TestMetadataProxyHandlerCacheMixin(object):
 
         req = mock.Mock(path_info='/the_path', query_string='', headers=hdrs,
                         method=method, body=body)
-        resp = mock.MagicMock(status=response_code)
+        resp = mock.MagicMock(status_code=response_code)
+        resp.content = 'content'
         req.response = resp
         with mock.patch.object(self.handler, '_sign_instance_id') as sign:
             sign.return_value = 'signed'
-            with mock.patch('httplib2.Http') as mock_http:
+            with mock.patch('requests.request') as mock_request:
-                resp.__getitem__.return_value = "text/plain"
+                resp.headers = {'content-type': 'text/plain'}
-                mock_http.return_value.request.return_value = (resp, 'content')
+                mock_request.return_value = resp
-
                 retval = self.handler._proxy_request('the_id', 'tenant_id',
                                                      req)
-                mock_http.assert_called_once_with(
+                mock_request.assert_called_once_with(
-                    ca_certs=None, disable_ssl_certificate_validation=True)
+                    method=method, url='http://9.9.9.9:8775/the_path',
-                mock_http.assert_has_calls([
+                    headers={
-                    mock.call().add_certificate(
+                        'X-Forwarded-For': '8.8.8.8',
-                        self.fake_conf.nova_client_priv_key,
+                        'X-Instance-ID-Signature': 'signed',
-                        self.fake_conf.nova_client_cert,
+                        'X-Instance-ID': 'the_id',
-                        "%s:%s" % (self.fake_conf.nova_metadata_host,
+                        'X-Tenant-ID': 'tenant_id'
-                                   self.fake_conf.nova_metadata_port)
+                    },
-                    ),
+                    data=body,
-                    mock.call().request(
+                    cert=(self.fake_conf.nova_client_cert,
-                        'http://9.9.9.9:8775/the_path',
+                          self.fake_conf.nova_client_priv_key),
-                        method=method,
+                    verify=False)
-                        headers={
-                            'X-Forwarded-For': '8.8.8.8',
-                            'X-Instance-ID-Signature': 'signed',
-                            'X-Instance-ID': 'the_id',
-                            'X-Tenant-ID': 'tenant_id'
-                        },
-                        body=body
-                    )]
-                )
 
                 return retval
 

requirements.txt

@@ -10,6 +10,7 @@ debtcollector>=1.2.0 # Apache-2.0
 eventlet!=0.18.3,!=0.20.1,>=0.18.2 # MIT
 pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.1.1 # BSD
 httplib2>=0.9.1 # MIT
+requests>=2.14.2 # Apache-2.0
 Jinja2>=2.10 # BSD License (3 clause)
 keystonemiddleware>=4.17.0 # Apache-2.0
 netaddr>=0.7.18 # BSD