NSX-v3 http read timeout

This patch exposes a new config property named
http_read_timeout in the nsx_v3 group of nsx.ini
which allows users to configure a HTTP read timeout
separately from http connect timeout (set via http_timeout).
With this addition users can provide finer grained timeout
settings for their deployed env.

A unit test is also included.

backport: liberty

Change-Id: I6bcb142329e348c7b2224daf32bd3f648ff2af77
Closes-Bug: #1540462

etc/nsx.ini

@@ -355,9 +355,12 @@
 # default system root CAs will be used.
 # insecure = True
 
-# The time in seconds before aborting an HTTP request to a NSX manager.
+# The time in seconds before aborting a HTTP connection to a NSX manager.
 # http_timeout = 75
 
+# The time in seconds before aborting a HTTP read response from a NSX manager.
+# http_read_timeout = 180
+
 # Maxiumum number of connection connections to each NSX manager.
 # concurrent_connections = 10
 

vmware_nsx/common/config.py

@@ -218,8 +218,12 @@ nsx_v3_opts = [
                        'system root CAs will be used.')),
     cfg.IntOpt('http_timeout',
                default=75,
-               help=_('Time before aborting a HTTP request to a '
+               help=_('Time before aborting a HTTP connection to a '
                       'NSX manager.')),
+    cfg.IntOpt('http_read_timeout',
+               default=180,
+               help=_('The time in seconds before aborting a HTTP read '
+                      'response from a NSX manager.')),
     cfg.IntOpt('concurrent_connections', default=10,
                help=_("Maximum concurrent connections to each NSX "
                       "manager.")),

vmware_nsx/nsxlib/v3/cluster.py

@@ -77,15 +77,16 @@ class TimeoutSession(requests.Session):
     at the session level.
     """
 
-    def __init__(self, timeout=cfg.CONF.nsx_v3.http_timeout):
+    def __init__(self, timeout=None, read_timeout=None):
-        self.timeout = timeout
+        self.timeout = timeout or cfg.CONF.nsx_v3.http_timeout
+        self.read_timeout = read_timeout or cfg.CONF.nsx_v3.http_read_timeout
         super(TimeoutSession, self).__init__()
 
     # wrapper timeouts at the session level
     # see: https://goo.gl/xNk7aM
     def request(self, *args, **kwargs):
         if 'timeout' not in kwargs:
-            kwargs['timeout'] = self.timeout
+            kwargs['timeout'] = (self.timeout, self.read_timeout)
         return super(TimeoutSession, self).request(*args, **kwargs)
 
 
@@ -109,7 +110,8 @@ class NSXRequestsHTTPProvider(AbstractHTTPProvider):
                 manager=endpoint.provider.url, operation=msg)
 
     def new_connection(self, cluster_api, provider):
-        session = TimeoutSession(cluster_api.http_timeout)
+        session = TimeoutSession(cluster_api.http_timeout,
+                                 cluster_api.http_read_timeout)
         session.auth = (cluster_api.username, cluster_api.password)
         # NSX v3 doesn't use redirects
         session.max_redirects = 0
@@ -430,6 +432,7 @@ class NSXClusteredAPI(ClusteredAPI):
                  ca_file=None,
                  concurrent_connections=None,
                  http_timeout=None,
+                 http_read_timeout=None,
                  conn_idle_timeout=None,
                  http_provider=None):
         self.username = username or cfg.CONF.nsx_v3.nsx_api_user
@@ -440,6 +443,8 @@ class NSXClusteredAPI(ClusteredAPI):
         self.conns_per_pool = (concurrent_connections or
                                cfg.CONF.nsx_v3.concurrent_connections)
         self.http_timeout = http_timeout or cfg.CONF.nsx_v3.http_timeout
+        self.http_read_timeout = (http_read_timeout or
+                                  cfg.CONF.nsx_v3.http_read_timeout)
         self.conn_idle_timeout = (conn_idle_timeout or
                                   cfg.CONF.nsx_v3.conn_idle_timeout)
 

vmware_nsx/tests/unit/nsxlib/v3/nsxlib_testcase.py

@@ -27,6 +27,8 @@ NSX_PASSWORD = 'default'
 NSX_MANAGER = '1.2.3.4'
 NSX_INSECURE = False
 NSX_CERT = '/opt/stack/certs/nsx.pem'
+NSX_HTTP_TIMEOUT = 10
+NSX_HTTP_READ_TIMEOUT = 180
 
 V3_CLIENT_PKG = 'vmware_nsx.nsxlib.v3.client'
 BRIDGE_FNS = ['create_resource', 'delete_resource',
@@ -44,6 +46,9 @@ class NsxLibTestCase(unittest.TestCase):
         cfg.CONF.set_override('nsx_api_managers', [NSX_MANAGER], 'nsx_v3')
         cfg.CONF.set_override('insecure', NSX_INSECURE, 'nsx_v3')
         cfg.CONF.set_override('ca_file', NSX_CERT, 'nsx_v3')
+        cfg.CONF.set_override('http_timeout', NSX_HTTP_TIMEOUT, 'nsx_v3')
+        cfg.CONF.set_override('http_read_timeout',
+                              NSX_HTTP_READ_TIMEOUT, 'nsx_v3')
         cfg.CONF.set_override('network_scheduler_driver',
             'neutron.scheduler.dhcp_agent_scheduler.AZAwareWeightScheduler')
 
@@ -112,7 +117,6 @@ class NsxClientTestCase(NsxLibTestCase):
             checked_kwargs = copy.copy(kwargs)
             del checked_kwargs['proxies']
             del checked_kwargs['stream']
-            del checked_kwargs['timeout']
             if 'allow_redirects' in checked_kwargs:
                 del checked_kwargs['allow_redirects']
 

vmware_nsx/tests/unit/nsxlib/v3/test_client.py

@@ -41,7 +41,9 @@ def _headers(**kwargs):
 
 def assert_call(verb, client_or_resource,
                 url, verify=nsxlib_testcase.NSX_CERT,
-                data=None, headers=DFT_ACCEPT_HEADERS):
+                data=None, headers=DFT_ACCEPT_HEADERS,
+                timeout=(nsxlib_testcase.NSX_HTTP_TIMEOUT,
+                         nsxlib_testcase.NSX_HTTP_READ_TIMEOUT)):
     nsx_client = client_or_resource
     if getattr(nsx_client, '_client', None) is not None:
         nsx_client = nsx_client._client
@@ -49,7 +51,7 @@ def assert_call(verb, client_or_resource,
     cluster.assert_called_once(
         verb,
         **{'url': url, 'verify': verify, 'body': data,
-           'headers': headers, 'cert': None})
+           'headers': headers, 'cert': None, 'timeout': timeout})
 
 
 def assert_json_call(verb, client_or_resource, url,

vmware_nsx/tests/unit/nsxlib/v3/test_cluster.py

@@ -127,6 +127,18 @@ class NsxV3ClusteredAPITestCase(nsxlib_testcase.NsxClientTestCase):
         for ep_id, ep in api.endpoints.items():
             self.assertEqual(ep.pool.max_size, 11)
 
+    def test_timeouts(self):
+        cfg.CONF.set_override(
+            'http_read_timeout', 37, 'nsx_v3')
+        cfg.CONF.set_override(
+            'http_timeout', 7, 'nsx_v3')
+
+        api = self.mock_nsx_clustered_api()
+        api.get('logical-ports')
+        mock_call = api.recorded_calls.method_calls[0]
+        name, args, kwargs = mock_call
+        self.assertEqual(kwargs['timeout'], (7, 37))
+
 
 class ClusteredAPITestCase(nsxlib_testcase.NsxClientTestCase):