x
/
vmware-nsxlib
Code Issues Proposed changes

Handle cluster connection closed by server 

When established connection is closed by server, endpoint should
not go down, and request should be retried with another connection.
However, if connection fails to be established, endpoint should go
down as coded before.

We observe server closing connections in spite of keep-alive with
policy endpoint.

Change-Id: I264da0ad47c31c9875a4be35acd4c5c4c88f4916
This commit is contained in:
Anna Khmelnitsky 2018-04-18 16:35:21 -07:00
parent e483b2d70f
commit 4fa59845d4
2 changed files with 27 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
vmware_nsxlib/tests/unit/v3/test_cluster.py
View File

@ -211,7 +211,7 @@ class ClusteredAPITestCase(nsxlib_testcase.NsxClientTestCase):
self.assertRaises(nsxlib_exc.StaleRevision,
api.get, 'api/v1/transport-zones')
def test_cluster_proxy_connection_error(self):
def test_cluster_proxy_connection_establish_error(self):
def connect_timeout():
raise requests_exceptions.ConnectTimeout()
@ -221,6 +221,21 @@ class ClusteredAPITestCase(nsxlib_testcase.NsxClientTestCase):
self.assertRaises(nsxlib_exc.ServiceClusterUnavailable,
api.get, 'api/v1/transport-zones')
def test_cluster_proxy_connection_aborted(self):
def connect_timeout():
raise requests_exceptions.ConnectionError("Connection Aborted")
def all_good():
pass
# First call will cause connection aborted error, but next one
# should work
api = self.mock_nsx_clustered_api(session_response=[connect_timeout,
all_good])
api._validate = mock.Mock()
self.assertEqual(cluster.ClusterHealth.GREEN, api.health)
def test_cluster_round_robin_servicing(self):
conf_managers = ['8.9.10.11', '9.10.11.12', '10.11.12.13']
api = self.mock_nsx_clustered_api(nsx_api_managers=conf_managers)

14
vmware_nsxlib/v3/cluster.py
View File

@ -221,6 +221,9 @@ class NSXRequestsHTTPProvider(AbstractHTTPProvider):
def is_connection_exception(self, exception):
return isinstance(exception, requests_exceptions.ConnectionError)
def is_conn_open_exception(self, exception):
return isinstance(exception, requests_exceptions.ConnectTimeout)
def get_default_headers(self, session, provider, allow_overwrite_header):
"""Get the default headers that should be added to future requests"""
session.default_headers = {}
@ -584,10 +587,15 @@ class ClusteredAPI(object):
if not self._http_provider.is_connection_exception(e):
# only trap and retry connection errors
raise e
endpoint.set_state(EndpointState.DOWN)
if self._http_provider.is_conn_open_exception(e):
# unable to establish new connection - endpoint is
# inaccessible
endpoint.set_state(EndpointState.DOWN)
LOG.debug("Connection to %s failed, checking additional "
"endpoints" % url)
# retry until exhausting endpoints
"connections and endpoints" % url)
# this might be a result of server closing connection
# retry until exhausting connections and endpoints
return self._proxy(proxy_for, uri, *args, **kwargs)