371 lines
14 KiB
Python
371 lines
14 KiB
Python
# Copyright 2012 OpenStack Foundation
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
# Originally copied from python-glanceclient
|
|
|
|
import copy
|
|
import hashlib
|
|
import httplib
|
|
import json
|
|
import OpenSSL
|
|
import posixpath
|
|
import re
|
|
from six import moves
|
|
import socket
|
|
import StringIO
|
|
import struct
|
|
import urlparse
|
|
|
|
from tempest import exceptions as exc
|
|
from tempest.openstack.common import log as logging
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
USER_AGENT = 'tempest'
|
|
CHUNKSIZE = 1024 * 64 # 64kB
|
|
TOKEN_CHARS_RE = re.compile('^[-A-Za-z0-9+/=]*$')
|
|
|
|
|
|
class HTTPClient(object):
|
|
|
|
def __init__(self, auth_provider, filters, **kwargs):
|
|
self.auth_provider = auth_provider
|
|
self.filters = filters
|
|
self.endpoint = auth_provider.base_url(filters)
|
|
endpoint_parts = urlparse.urlparse(self.endpoint)
|
|
self.endpoint_scheme = endpoint_parts.scheme
|
|
self.endpoint_hostname = endpoint_parts.hostname
|
|
self.endpoint_port = endpoint_parts.port
|
|
self.endpoint_path = endpoint_parts.path
|
|
|
|
self.connection_class = self.get_connection_class(self.endpoint_scheme)
|
|
self.connection_kwargs = self.get_connection_kwargs(
|
|
self.endpoint_scheme, **kwargs)
|
|
|
|
@staticmethod
|
|
def get_connection_class(scheme):
|
|
if scheme == 'https':
|
|
return VerifiedHTTPSConnection
|
|
else:
|
|
return httplib.HTTPConnection
|
|
|
|
@staticmethod
|
|
def get_connection_kwargs(scheme, **kwargs):
|
|
_kwargs = {'timeout': float(kwargs.get('timeout', 600))}
|
|
|
|
if scheme == 'https':
|
|
_kwargs['cacert'] = kwargs.get('cacert', None)
|
|
_kwargs['cert_file'] = kwargs.get('cert_file', None)
|
|
_kwargs['key_file'] = kwargs.get('key_file', None)
|
|
_kwargs['insecure'] = kwargs.get('insecure', False)
|
|
_kwargs['ssl_compression'] = kwargs.get('ssl_compression', True)
|
|
|
|
return _kwargs
|
|
|
|
def get_connection(self):
|
|
_class = self.connection_class
|
|
try:
|
|
return _class(self.endpoint_hostname, self.endpoint_port,
|
|
**self.connection_kwargs)
|
|
except httplib.InvalidURL:
|
|
raise exc.EndpointNotFound
|
|
|
|
def _http_request(self, url, method, **kwargs):
|
|
"""Send an http request with the specified characteristics.
|
|
|
|
Wrapper around httplib.HTTP(S)Connection.request to handle tasks such
|
|
as setting headers and error handling.
|
|
"""
|
|
# Copy the kwargs so we can reuse the original in case of redirects
|
|
kwargs['headers'] = copy.deepcopy(kwargs.get('headers', {}))
|
|
kwargs['headers'].setdefault('User-Agent', USER_AGENT)
|
|
|
|
self._log_request(method, url, kwargs['headers'])
|
|
|
|
conn = self.get_connection()
|
|
|
|
try:
|
|
url_parts = urlparse.urlparse(url)
|
|
conn_url = posixpath.normpath(url_parts.path)
|
|
LOG.debug('Actual Path: {path}'.format(path=conn_url))
|
|
if kwargs['headers'].get('Transfer-Encoding') == 'chunked':
|
|
conn.putrequest(method, conn_url)
|
|
for header, value in kwargs['headers'].items():
|
|
conn.putheader(header, value)
|
|
conn.endheaders()
|
|
chunk = kwargs['body'].read(CHUNKSIZE)
|
|
# Chunk it, baby...
|
|
while chunk:
|
|
conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
|
|
chunk = kwargs['body'].read(CHUNKSIZE)
|
|
conn.send('0\r\n\r\n')
|
|
else:
|
|
conn.request(method, conn_url, **kwargs)
|
|
resp = conn.getresponse()
|
|
except socket.gaierror as e:
|
|
message = ("Error finding address for %(url)s: %(e)s" %
|
|
{'url': url, 'e': e})
|
|
raise exc.EndpointNotFound(message)
|
|
except (socket.error, socket.timeout) as e:
|
|
message = ("Error communicating with %(endpoint)s %(e)s" %
|
|
{'endpoint': self.endpoint, 'e': e})
|
|
raise exc.TimeoutException(message)
|
|
|
|
body_iter = ResponseBodyIterator(resp)
|
|
# Read body into string if it isn't obviously image data
|
|
if resp.getheader('content-type', None) != 'application/octet-stream':
|
|
body_str = ''.join([body_chunk for body_chunk in body_iter])
|
|
body_iter = StringIO.StringIO(body_str)
|
|
self._log_response(resp, None)
|
|
else:
|
|
self._log_response(resp, body_iter)
|
|
|
|
return resp, body_iter
|
|
|
|
def _log_request(self, method, url, headers):
|
|
LOG.info('Request: ' + method + ' ' + url)
|
|
if headers:
|
|
headers_out = headers
|
|
if 'X-Auth-Token' in headers and headers['X-Auth-Token']:
|
|
token = headers['X-Auth-Token']
|
|
if len(token) > 64 and TOKEN_CHARS_RE.match(token):
|
|
headers_out = headers.copy()
|
|
headers_out['X-Auth-Token'] = "<Token omitted>"
|
|
LOG.info('Request Headers: ' + str(headers_out))
|
|
|
|
def _log_response(self, resp, body):
|
|
status = str(resp.status)
|
|
LOG.info("Response Status: " + status)
|
|
if resp.getheaders():
|
|
LOG.info('Response Headers: ' + str(resp.getheaders()))
|
|
if body:
|
|
str_body = str(body)
|
|
length = len(body)
|
|
LOG.info('Response Body: ' + str_body[:2048])
|
|
if length >= 2048:
|
|
self.LOG.debug("Large body (%d) md5 summary: %s", length,
|
|
hashlib.md5(str_body).hexdigest())
|
|
|
|
def json_request(self, method, url, **kwargs):
|
|
kwargs.setdefault('headers', {})
|
|
kwargs['headers'].setdefault('Content-Type', 'application/json')
|
|
|
|
if 'body' in kwargs:
|
|
kwargs['body'] = json.dumps(kwargs['body'])
|
|
|
|
resp, body_iter = self._http_request(url, method, **kwargs)
|
|
|
|
if 'application/json' in resp.getheader('content-type', ''):
|
|
body = ''.join([chunk for chunk in body_iter])
|
|
try:
|
|
body = json.loads(body)
|
|
except ValueError:
|
|
LOG.error('Could not decode response body as JSON')
|
|
else:
|
|
body = None
|
|
|
|
return resp, body
|
|
|
|
def raw_request(self, method, url, **kwargs):
|
|
kwargs.setdefault('headers', {})
|
|
kwargs['headers'].setdefault('Content-Type',
|
|
'application/octet-stream')
|
|
if 'body' in kwargs:
|
|
if (hasattr(kwargs['body'], 'read')
|
|
and method.lower() in ('post', 'put')):
|
|
# We use 'Transfer-Encoding: chunked' because
|
|
# body size may not always be known in advance.
|
|
kwargs['headers']['Transfer-Encoding'] = 'chunked'
|
|
|
|
# Decorate the request with auth
|
|
req_url, kwargs['headers'], kwargs['body'] = \
|
|
self.auth_provider.auth_request(
|
|
method=method, url=url, headers=kwargs['headers'],
|
|
body=kwargs.get('body', None), filters=self.filters)
|
|
return self._http_request(req_url, method, **kwargs)
|
|
|
|
|
|
class OpenSSLConnectionDelegator(object):
|
|
"""
|
|
An OpenSSL.SSL.Connection delegator.
|
|
|
|
Supplies an additional 'makefile' method which httplib requires
|
|
and is not present in OpenSSL.SSL.Connection.
|
|
|
|
Note: Since it is not possible to inherit from OpenSSL.SSL.Connection
|
|
a delegator must be used.
|
|
"""
|
|
def __init__(self, *args, **kwargs):
|
|
self.connection = OpenSSL.SSL.Connection(*args, **kwargs)
|
|
|
|
def __getattr__(self, name):
|
|
return getattr(self.connection, name)
|
|
|
|
def makefile(self, *args, **kwargs):
|
|
# Ensure the socket is closed when this file is closed
|
|
kwargs['close'] = True
|
|
return socket._fileobject(self.connection, *args, **kwargs)
|
|
|
|
|
|
class VerifiedHTTPSConnection(httplib.HTTPSConnection):
|
|
"""
|
|
Extended HTTPSConnection which uses the OpenSSL library
|
|
for enhanced SSL support.
|
|
Note: Much of this functionality can eventually be replaced
|
|
with native Python 3.3 code.
|
|
"""
|
|
def __init__(self, host, port=None, key_file=None, cert_file=None,
|
|
cacert=None, timeout=None, insecure=False,
|
|
ssl_compression=True):
|
|
httplib.HTTPSConnection.__init__(self, host, port,
|
|
key_file=key_file,
|
|
cert_file=cert_file)
|
|
self.key_file = key_file
|
|
self.cert_file = cert_file
|
|
self.timeout = timeout
|
|
self.insecure = insecure
|
|
self.ssl_compression = ssl_compression
|
|
self.cacert = cacert
|
|
self.setcontext()
|
|
|
|
@staticmethod
|
|
def host_matches_cert(host, x509):
|
|
"""
|
|
Verify that the the x509 certificate we have received
|
|
from 'host' correctly identifies the server we are
|
|
connecting to, ie that the certificate's Common Name
|
|
or a Subject Alternative Name matches 'host'.
|
|
"""
|
|
# First see if we can match the CN
|
|
if x509.get_subject().commonName == host:
|
|
return True
|
|
|
|
# Also try Subject Alternative Names for a match
|
|
san_list = None
|
|
for i in moves.xrange(x509.get_extension_count()):
|
|
ext = x509.get_extension(i)
|
|
if ext.get_short_name() == 'subjectAltName':
|
|
san_list = str(ext)
|
|
for san in ''.join(san_list.split()).split(','):
|
|
if san == "DNS:%s" % host:
|
|
return True
|
|
|
|
# Server certificate does not match host
|
|
msg = ('Host "%s" does not match x509 certificate contents: '
|
|
'CommonName "%s"' % (host, x509.get_subject().commonName))
|
|
if san_list is not None:
|
|
msg = msg + ', subjectAltName "%s"' % san_list
|
|
raise exc.SSLCertificateError(msg)
|
|
|
|
def verify_callback(self, connection, x509, errnum,
|
|
depth, preverify_ok):
|
|
if x509.has_expired():
|
|
msg = "SSL Certificate expired on '%s'" % x509.get_notAfter()
|
|
raise exc.SSLCertificateError(msg)
|
|
|
|
if depth == 0 and preverify_ok is True:
|
|
# We verify that the host matches against the last
|
|
# certificate in the chain
|
|
return self.host_matches_cert(self.host, x509)
|
|
else:
|
|
# Pass through OpenSSL's default result
|
|
return preverify_ok
|
|
|
|
def setcontext(self):
|
|
"""
|
|
Set up the OpenSSL context.
|
|
"""
|
|
self.context = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
|
|
|
|
if self.ssl_compression is False:
|
|
self.context.set_options(0x20000) # SSL_OP_NO_COMPRESSION
|
|
|
|
if self.insecure is not True:
|
|
self.context.set_verify(OpenSSL.SSL.VERIFY_PEER,
|
|
self.verify_callback)
|
|
else:
|
|
self.context.set_verify(OpenSSL.SSL.VERIFY_NONE,
|
|
self.verify_callback)
|
|
|
|
if self.cert_file:
|
|
try:
|
|
self.context.use_certificate_file(self.cert_file)
|
|
except Exception as e:
|
|
msg = 'Unable to load cert from "%s" %s' % (self.cert_file, e)
|
|
raise exc.SSLConfigurationError(msg)
|
|
if self.key_file is None:
|
|
# We support having key and cert in same file
|
|
try:
|
|
self.context.use_privatekey_file(self.cert_file)
|
|
except Exception as e:
|
|
msg = ('No key file specified and unable to load key '
|
|
'from "%s" %s' % (self.cert_file, e))
|
|
raise exc.SSLConfigurationError(msg)
|
|
|
|
if self.key_file:
|
|
try:
|
|
self.context.use_privatekey_file(self.key_file)
|
|
except Exception as e:
|
|
msg = 'Unable to load key from "%s" %s' % (self.key_file, e)
|
|
raise exc.SSLConfigurationError(msg)
|
|
|
|
if self.cacert:
|
|
try:
|
|
self.context.load_verify_locations(self.cacert)
|
|
except Exception as e:
|
|
msg = 'Unable to load CA from "%s"' % (self.cacert, e)
|
|
raise exc.SSLConfigurationError(msg)
|
|
else:
|
|
self.context.set_default_verify_paths()
|
|
|
|
def connect(self):
|
|
"""
|
|
Connect to an SSL port using the OpenSSL library and apply
|
|
per-connection parameters.
|
|
"""
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
if self.timeout is not None:
|
|
# '0' microseconds
|
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVTIMEO,
|
|
struct.pack('LL', self.timeout, 0))
|
|
self.sock = OpenSSLConnectionDelegator(self.context, sock)
|
|
self.sock.connect((self.host, self.port))
|
|
|
|
def close(self):
|
|
if self.sock:
|
|
# Remove the reference to the socket but don't close it yet.
|
|
# Response close will close both socket and associated
|
|
# file. Closing socket too soon will cause response
|
|
# reads to fail with socket IO error 'Bad file descriptor'.
|
|
self.sock = None
|
|
httplib.HTTPSConnection.close(self)
|
|
|
|
|
|
class ResponseBodyIterator(object):
|
|
"""A class that acts as an iterator over an HTTP response."""
|
|
|
|
def __init__(self, resp):
|
|
self.resp = resp
|
|
|
|
def __iter__(self):
|
|
while True:
|
|
yield self.next()
|
|
|
|
def next(self):
|
|
chunk = self.resp.read(CHUNKSIZE)
|
|
if chunk:
|
|
return chunk
|
|
else:
|
|
raise StopIteration()
|