Set timeout value in urllib3.poolmanager.PoolManager

If timeout is not set in urllib3.poolmanager.PoolManager, it will
use "Python's default timeout for sockets", but if "timeout for
sockets" is not set, the timeout will be infinite(will not timeout).
so this is intented to set timeout value in urllib3.poolmanager.
PoolManager to avoid infinite timeout.

Change-Id: Ic035fdb93734c926b26b33feb610e0977e48c646
Closes-Bug: #1558931

releasenotes/notes/add-httptimeout-in-restclient-ax78061900e3f3d7.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - RestClient now supports setting timeout in urllib3.poolmanager.
+    Clients will use CONF.service_clients.http_timeout for timeout
+    value to wait for http request to response.
+  - KeystoneAuthProvider will accept http_timeout and will use it in
+    get_credentials.

tempest/config.py

@@ -173,6 +173,16 @@ IdentityGroup = [
                      "a domain scoped token to use admin APIs")
 ]
 
+service_clients_group = cfg.OptGroup(name='service-clients',
+                                     title="Service Clients Options")
+
+ServiceClientsGroup = [
+    cfg.IntOpt('http_timeout',
+               default=60,
+               help='Timeout in seconds to wait for the http request to '
+                    'return'),
+]
+
 identity_feature_group = cfg.OptGroup(name='identity-feature-enabled',
                                       title='Enabled Identity Features')
 
@@ -1119,6 +1129,7 @@ _opts = [
     (compute_group, ComputeGroup),
     (compute_features_group, ComputeFeaturesGroup),
     (identity_group, IdentityGroup),
+    (service_clients_group, ServiceClientsGroup),
     (identity_feature_group, IdentityFeatureGroup),
     (image_group, ImageGroup),
     (image_feature_group, ImageFeaturesGroup),
@@ -1184,6 +1195,7 @@ class TempestConfigPrivate(object):
         self.compute = _CONF.compute
         self.compute_feature_enabled = _CONF['compute-feature-enabled']
         self.identity = _CONF.identity
+        self.service_clients = _CONF['service-clients']
         self.identity_feature_enabled = _CONF['identity-feature-enabled']
         self.image = _CONF.image
         self.image_feature_enabled = _CONF['image-feature-enabled']
@@ -1372,6 +1384,7 @@ def service_client_config(service_client_name=None):
         * `disable_ssl_certificate_validation`
         * `ca_certs`
         * `trace_requests`
+        * `http_timeout`
 
     The dict returned by this does not fit a few service clients:
 
@@ -1393,7 +1406,8 @@ def service_client_config(service_client_name=None):
         'disable_ssl_certificate_validation':
             CONF.identity.disable_ssl_certificate_validation,
         'ca_certs': CONF.identity.ca_certificates_file,
-        'trace_requests': CONF.debug.trace_requests
+        'trace_requests': CONF.debug.trace_requests,
+        'http_timeout': CONF.service_clients.http_timeout
     }
 
     if service_client_name is None:

tempest/lib/auth.py

@@ -260,11 +260,13 @@ class KeystoneAuthProvider(AuthProvider):
 
     def __init__(self, credentials, auth_url,
                  disable_ssl_certificate_validation=None,
-                 ca_certs=None, trace_requests=None, scope='project'):
+                 ca_certs=None, trace_requests=None, scope='project',
+                 http_timeout=None):
         super(KeystoneAuthProvider, self).__init__(credentials, scope)
         self.dscv = disable_ssl_certificate_validation
         self.ca_certs = ca_certs
         self.trace_requests = trace_requests
+        self.http_timeout = http_timeout
         self.auth_url = auth_url
         self.auth_client = self._auth_client(auth_url)
 
@@ -342,7 +344,8 @@ class KeystoneV2AuthProvider(KeystoneAuthProvider):
     def _auth_client(self, auth_url):
         return json_v2id.TokenClient(
             auth_url, disable_ssl_certificate_validation=self.dscv,
-            ca_certs=self.ca_certs, trace_requests=self.trace_requests)
+            ca_certs=self.ca_certs, trace_requests=self.trace_requests,
+            http_timeout=self.http_timeout)
 
     def _auth_params(self):
         """Auth parameters to be passed to the token request
@@ -429,7 +432,8 @@ class KeystoneV3AuthProvider(KeystoneAuthProvider):
     def _auth_client(self, auth_url):
         return json_v3id.V3TokenClient(
             auth_url, disable_ssl_certificate_validation=self.dscv,
-            ca_certs=self.ca_certs, trace_requests=self.trace_requests)
+            ca_certs=self.ca_certs, trace_requests=self.trace_requests,
+            http_timeout=self.http_timeout)
 
     def _auth_params(self):
         """Auth parameters to be passed to the token request
@@ -595,7 +599,7 @@ def is_identity_version_supported(identity_version):
 
 def get_credentials(auth_url, fill_in=True, identity_version='v2',
                     disable_ssl_certificate_validation=None, ca_certs=None,
-                    trace_requests=None, **kwargs):
+                    trace_requests=None, http_timeout=None, **kwargs):
     """Builds a credentials object based on the configured auth_version
 
     :param auth_url (string): Full URI of the OpenStack Identity API(Keystone)
@@ -611,6 +615,8 @@ def get_credentials(auth_url, fill_in=True, identity_version='v2',
     :param ca_certs: CA certificate bundle for validation of certificates
            in SSL API requests to the auth system
     :param trace_requests: trace in log API requests to the auth system
+    :param http_timeout: timeout in seconds to wait for the http request to
+           return
     :param kwargs (dict): Dict of credential key/value pairs
 
     Examples:
@@ -634,7 +640,8 @@ def get_credentials(auth_url, fill_in=True, identity_version='v2',
         dscv = disable_ssl_certificate_validation
         auth_provider = auth_provider_class(
             creds, auth_url, disable_ssl_certificate_validation=dscv,
-            ca_certs=ca_certs, trace_requests=trace_requests)
+            ca_certs=ca_certs, trace_requests=trace_requests,
+            http_timeout=http_timeout)
         creds = auth_provider.fill_credentials()
     return creds
 

tempest/lib/common/http.py

@@ -18,7 +18,7 @@ import urllib3
 
 class ClosingHttp(urllib3.poolmanager.PoolManager):
     def __init__(self, disable_ssl_certificate_validation=False,
-                 ca_certs=None):
+                 ca_certs=None, timeout=None):
         kwargs = {}
 
         if disable_ssl_certificate_validation:
@@ -29,6 +29,9 @@ class ClosingHttp(urllib3.poolmanager.PoolManager):
             kwargs['cert_reqs'] = 'CERT_REQUIRED'
             kwargs['ca_certs'] = ca_certs
 
+        if timeout:
+            kwargs['timeout'] = timeout
+
         super(ClosingHttp, self).__init__(**kwargs)
 
     def request(self, url, method, *args, **kwargs):

tempest/lib/common/rest_client.py

@@ -66,6 +66,8 @@ class RestClient(object):
                          TLS server cert
     :param str trace_request: Regex to use for specifying logging the entirety
                               of the request and response payload
+    :param str http_timeout: Timeout in seconds to wait for the http request to
+                             return
     """
     TYPE = "json"
 
@@ -78,7 +80,7 @@ class RestClient(object):
                  endpoint_type='publicURL',
                  build_interval=1, build_timeout=60,
                  disable_ssl_certificate_validation=False, ca_certs=None,
-                 trace_requests='', name=None):
+                 trace_requests='', name=None, http_timeout=None):
         self.auth_provider = auth_provider
         self.service = service
         self.region = region
@@ -99,7 +101,8 @@ class RestClient(object):
                                        'vary', 'www-authenticate'))
         dscv = disable_ssl_certificate_validation
         self.http_obj = http.ClosingHttp(
-            disable_ssl_certificate_validation=dscv, ca_certs=ca_certs)
+            disable_ssl_certificate_validation=dscv, ca_certs=ca_certs,
+            timeout=http_timeout)
 
     def _get_type(self):
         return self.TYPE

tempest/lib/services/identity/v2/token_client.py

@@ -22,11 +22,11 @@ from tempest.lib import exceptions
 class TokenClient(rest_client.RestClient):
 
     def __init__(self, auth_url, disable_ssl_certificate_validation=None,
-                 ca_certs=None, trace_requests=None):
+                 ca_certs=None, trace_requests=None, **kwargs):
         dscv = disable_ssl_certificate_validation
         super(TokenClient, self).__init__(
             None, None, None, disable_ssl_certificate_validation=dscv,
-            ca_certs=ca_certs, trace_requests=trace_requests)
+            ca_certs=ca_certs, trace_requests=trace_requests, **kwargs)
 
         if auth_url is None:
             raise exceptions.IdentityError("Couldn't determine auth_url")

tempest/lib/services/identity/v3/token_client.py

@@ -22,11 +22,11 @@ from tempest.lib import exceptions
 class V3TokenClient(rest_client.RestClient):
 
     def __init__(self, auth_url, disable_ssl_certificate_validation=None,
-                 ca_certs=None, trace_requests=None):
+                 ca_certs=None, trace_requests=None, **kwargs):
         dscv = disable_ssl_certificate_validation
         super(V3TokenClient, self).__init__(
             None, None, None, disable_ssl_certificate_validation=dscv,
-            ca_certs=ca_certs, trace_requests=trace_requests)
+            ca_certs=ca_certs, trace_requests=trace_requests, **kwargs)
 
         if auth_url is None:
             raise exceptions.IdentityError("Couldn't determine auth_url")