Add config options to set proxy_url

As part of the migration from httblib2 -> urllib3 we lost support for
handling the env vars for setting an http proxy. This wasn't ever an
explicit feature in tempest, but instead was an artifact of our
library choice for doing http. However as people relied on this
functionality and it is a useful feature having tempest support this
explicitly will make sure we don't accidentally drop support for it in
the future. This commit adds a new config options to specify and
proxy_url to replace the lost functionality.

Change-Id: Id8f6422a323f8bfdb10527f55c0cb046622b88bf
Closes-Bug: #1556864
This commit is contained in:
Matthew Treinish 2016-09-01 11:44:57 -04:00 committed by Ken'ichi Ohmichi
parent 46a0fa7cec
commit 74514400a2
6 changed files with 84 additions and 13 deletions

View File

@ -0,0 +1,9 @@
---
features:
- Adds a new config options, ``proxy_url``. This options is used to configure
running tempest through a proxy server.
- The RestClient class in tempest.lib.rest_client has a new kwarg parameters,
``proxy_url``, that is used to set a proxy server.
- A new class was added to tempest.lib.http, ClosingProxyHttp. This behaves
identically to ClosingHttp except that it requires a proxy url and will
establish a connection through a proxy

View File

@ -194,6 +194,8 @@ ServiceClientsGroup = [
default=60, default=60,
help='Timeout in seconds to wait for the http request to ' help='Timeout in seconds to wait for the http request to '
'return'), 'return'),
cfg.StrOpt('proxy_url',
help='Specify an http proxy to use.')
] ]
identity_feature_group = cfg.OptGroup(name='identity-feature-enabled', identity_feature_group = cfg.OptGroup(name='identity-feature-enabled',
@ -1308,6 +1310,7 @@ def service_client_config(service_client_name=None):
* `ca_certs` * `ca_certs`
* `trace_requests` * `trace_requests`
* `http_timeout` * `http_timeout`
* `proxy_url`
The dict returned by this does not fit a few service clients: The dict returned by this does not fit a few service clients:
@ -1330,7 +1333,8 @@ def service_client_config(service_client_name=None):
CONF.identity.disable_ssl_certificate_validation, CONF.identity.disable_ssl_certificate_validation,
'ca_certs': CONF.identity.ca_certificates_file, 'ca_certs': CONF.identity.ca_certificates_file,
'trace_requests': CONF.debug.trace_requests, 'trace_requests': CONF.debug.trace_requests,
'http_timeout': CONF.service_clients.http_timeout 'http_timeout': CONF.service_clients.http_timeout,
'proxy_url': CONF.service_clients.proxy_url,
} }
if service_client_name is None: if service_client_name is None:

View File

@ -261,12 +261,13 @@ class KeystoneAuthProvider(AuthProvider):
def __init__(self, credentials, auth_url, def __init__(self, credentials, auth_url,
disable_ssl_certificate_validation=None, disable_ssl_certificate_validation=None,
ca_certs=None, trace_requests=None, scope='project', ca_certs=None, trace_requests=None, scope='project',
http_timeout=None): http_timeout=None, proxy_url=None):
super(KeystoneAuthProvider, self).__init__(credentials, scope) super(KeystoneAuthProvider, self).__init__(credentials, scope)
self.dscv = disable_ssl_certificate_validation self.dscv = disable_ssl_certificate_validation
self.ca_certs = ca_certs self.ca_certs = ca_certs
self.trace_requests = trace_requests self.trace_requests = trace_requests
self.http_timeout = http_timeout self.http_timeout = http_timeout
self.proxy_url = proxy_url
self.auth_url = auth_url self.auth_url = auth_url
self.auth_client = self._auth_client(auth_url) self.auth_client = self._auth_client(auth_url)
@ -345,7 +346,7 @@ class KeystoneV2AuthProvider(KeystoneAuthProvider):
return json_v2id.TokenClient( return json_v2id.TokenClient(
auth_url, disable_ssl_certificate_validation=self.dscv, auth_url, disable_ssl_certificate_validation=self.dscv,
ca_certs=self.ca_certs, trace_requests=self.trace_requests, ca_certs=self.ca_certs, trace_requests=self.trace_requests,
http_timeout=self.http_timeout) http_timeout=self.http_timeout, proxy_url=self.proxy_url)
def _auth_params(self): def _auth_params(self):
"""Auth parameters to be passed to the token request """Auth parameters to be passed to the token request
@ -433,7 +434,7 @@ class KeystoneV3AuthProvider(KeystoneAuthProvider):
return json_v3id.V3TokenClient( return json_v3id.V3TokenClient(
auth_url, disable_ssl_certificate_validation=self.dscv, auth_url, disable_ssl_certificate_validation=self.dscv,
ca_certs=self.ca_certs, trace_requests=self.trace_requests, ca_certs=self.ca_certs, trace_requests=self.trace_requests,
http_timeout=self.http_timeout) http_timeout=self.http_timeout, proxy_url=self.proxy_url)
def _auth_params(self): def _auth_params(self):
"""Auth parameters to be passed to the token request """Auth parameters to be passed to the token request

View File

@ -17,6 +17,47 @@ import six
import urllib3 import urllib3
class ClosingProxyHttp(urllib3.ProxyManager):
def __init__(self, proxy_url, disable_ssl_certificate_validation=False,
ca_certs=None, timeout=None):
kwargs = {}
if disable_ssl_certificate_validation:
urllib3.disable_warnings()
kwargs['cert_reqs'] = 'CERT_NONE'
elif ca_certs:
kwargs['cert_reqs'] = 'CERT_REQUIRED'
kwargs['ca_certs'] = ca_certs
if timeout:
kwargs['timeout'] = timeout
super(ClosingProxyHttp, self).__init__(proxy_url, **kwargs)
def request(self, url, method, *args, **kwargs):
class Response(dict):
def __init__(self, info):
for key, value in info.getheaders().items():
self[key.lower()] = value
self.status = info.status
self['status'] = str(self.status)
self.reason = info.reason
self.version = info.version
self['content-location'] = url
original_headers = kwargs.get('headers', {})
new_headers = dict(original_headers, connection='close')
new_kwargs = dict(kwargs, headers=new_headers)
# Follow up to 5 redirections. Don't raise an exception if
# it's exceeded but return the HTTP 3XX response instead.
retry = urllib3.util.Retry(raise_on_redirect=False, redirect=5)
r = super(ClosingProxyHttp, self).request(method, url, retries=retry,
*args, **new_kwargs)
return Response(r), r.data
class ClosingHttp(urllib3.poolmanager.PoolManager): class ClosingHttp(urllib3.poolmanager.PoolManager):
def __init__(self, disable_ssl_certificate_validation=False, def __init__(self, disable_ssl_certificate_validation=False,
ca_certs=None, timeout=None): ca_certs=None, timeout=None):

View File

@ -69,6 +69,7 @@ class RestClient(object):
of the request and response payload of the request and response payload
:param str http_timeout: Timeout in seconds to wait for the http request to :param str http_timeout: Timeout in seconds to wait for the http request to
return return
:param str proxy_url: http proxy url to use.
""" """
# The version of the API this client implements # The version of the API this client implements
@ -80,7 +81,8 @@ class RestClient(object):
endpoint_type='publicURL', endpoint_type='publicURL',
build_interval=1, build_timeout=60, build_interval=1, build_timeout=60,
disable_ssl_certificate_validation=False, ca_certs=None, disable_ssl_certificate_validation=False, ca_certs=None,
trace_requests='', name=None, http_timeout=None): trace_requests='', name=None, http_timeout=None,
proxy_url=None):
self.auth_provider = auth_provider self.auth_provider = auth_provider
self.service = service self.service = service
self.region = region self.region = region
@ -100,9 +102,16 @@ class RestClient(object):
'retry-after', 'server', 'retry-after', 'server',
'vary', 'www-authenticate')) 'vary', 'www-authenticate'))
dscv = disable_ssl_certificate_validation dscv = disable_ssl_certificate_validation
self.http_obj = http.ClosingHttp(
disable_ssl_certificate_validation=dscv, ca_certs=ca_certs, if proxy_url:
timeout=http_timeout) self.http_obj = http.ClosingProxyHttp(
proxy_url,
disable_ssl_certificate_validation=dscv, ca_certs=ca_certs,
timeout=http_timeout)
else:
self.http_obj = http.ClosingHttp(
disable_ssl_certificate_validation=dscv, ca_certs=ca_certs,
timeout=http_timeout)
def get_headers(self, accept_type=None, send_type=None): def get_headers(self, accept_type=None, send_type=None):
"""Return the default headers which will be used with outgoing requests """Return the default headers which will be used with outgoing requests

View File

@ -276,7 +276,7 @@ class ServiceClients(object):
@removals.removed_kwarg('client_parameters') @removals.removed_kwarg('client_parameters')
def __init__(self, credentials, identity_uri, region=None, scope='project', def __init__(self, credentials, identity_uri, region=None, scope='project',
disable_ssl_certificate_validation=True, ca_certs=None, disable_ssl_certificate_validation=True, ca_certs=None,
trace_requests='', client_parameters=None): trace_requests='', client_parameters=None, proxy_url=None):
"""Service Clients provider """Service Clients provider
Instantiate a `ServiceClients` object, from a set of credentials and an Instantiate a `ServiceClients` object, from a set of credentials and an
@ -336,6 +336,8 @@ class ServiceClients(object):
name, as declared in `service_clients.available_modules()` except name, as declared in `service_clients.available_modules()` except
for the version. Values are dictionaries of parameters that are for the version. Values are dictionaries of parameters that are
going to be passed to all clients in the service client module. going to be passed to all clients in the service client module.
:param proxy_url: Applies to auth and to all service clients, set a
proxy url for the clients to use.
""" """
self._registered_services = set([]) self._registered_services = set([])
self.credentials = credentials self.credentials = credentials
@ -360,14 +362,18 @@ class ServiceClients(object):
self.dscv = disable_ssl_certificate_validation self.dscv = disable_ssl_certificate_validation
self.ca_certs = ca_certs self.ca_certs = ca_certs
self.trace_requests = trace_requests self.trace_requests = trace_requests
self.proxy_url = proxy_url
# Creates an auth provider for the credentials # Creates an auth provider for the credentials
self.auth_provider = auth_provider_class( self.auth_provider = auth_provider_class(
self.credentials, self.identity_uri, scope=scope, self.credentials, self.identity_uri, scope=scope,
disable_ssl_certificate_validation=self.dscv, disable_ssl_certificate_validation=self.dscv,
ca_certs=self.ca_certs, trace_requests=self.trace_requests) ca_certs=self.ca_certs, trace_requests=self.trace_requests,
proxy_url=proxy_url)
# Setup some defaults for client parameters of registered services # Setup some defaults for client parameters of registered services
client_parameters = client_parameters or {} client_parameters = client_parameters or {}
self.parameters = {} self.parameters = {}
# Parameters are provided for unversioned services # Parameters are provided for unversioned services
all_modules = available_modules() | _tempest_internal_modules() all_modules = available_modules() | _tempest_internal_modules()
unversioned_services = set( unversioned_services = set(
@ -420,8 +426,8 @@ class ServiceClients(object):
clients in tempest. clients in tempest.
:param client_names: List or set of names of service client classes. :param client_names: List or set of names of service client classes.
:param kwargs: Extra optional parameters to be passed to all clients. :param kwargs: Extra optional parameters to be passed to all clients.
ServiceClient provides defaults for region, dscv, ca_certs and ServiceClient provides defaults for region, dscv, ca_certs, http
trace_requests. proxies and trace_requests.
:raise ServiceClientRegistrationException: if the provided name is :raise ServiceClientRegistrationException: if the provided name is
already in use or if service_version is already registered. already in use or if service_version is already registered.
:raise ImportError: if module_path cannot be imported. :raise ImportError: if module_path cannot be imported.
@ -442,7 +448,8 @@ class ServiceClients(object):
params = dict(region=self.region, params = dict(region=self.region,
disable_ssl_certificate_validation=self.dscv, disable_ssl_certificate_validation=self.dscv,
ca_certs=self.ca_certs, ca_certs=self.ca_certs,
trace_requests=self.trace_requests) trace_requests=self.trace_requests,
proxy_url=self.proxy_url)
params.update(kwargs) params.update(kwargs)
# Instantiate the client factory # Instantiate the client factory
_factory = ClientsFactory(module_path=module_path, _factory = ClientsFactory(module_path=module_path,