From 4fa59845d4c7a88a6324de7c7c6b4672c2c5a1ad Mon Sep 17 00:00:00 2001 From: Anna Khmelnitsky Date: Wed, 18 Apr 2018 16:35:21 -0700 Subject: [PATCH] Handle cluster connection closed by server When established connection is closed by server, endpoint should not go down, and request should be retried with another connection. However, if connection fails to be established, endpoint should go down as coded before. We observe server closing connections in spite of keep-alive with policy endpoint. Change-Id: I264da0ad47c31c9875a4be35acd4c5c4c88f4916 --- vmware_nsxlib/tests/unit/v3/test_cluster.py | 17 ++++++++++++++++- vmware_nsxlib/v3/cluster.py | 14 +++++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/vmware_nsxlib/tests/unit/v3/test_cluster.py b/vmware_nsxlib/tests/unit/v3/test_cluster.py index 5b05c951..a3bc2c83 100644 --- a/vmware_nsxlib/tests/unit/v3/test_cluster.py +++ b/vmware_nsxlib/tests/unit/v3/test_cluster.py @@ -211,7 +211,7 @@ class ClusteredAPITestCase(nsxlib_testcase.NsxClientTestCase): self.assertRaises(nsxlib_exc.StaleRevision, api.get, 'api/v1/transport-zones') - def test_cluster_proxy_connection_error(self): + def test_cluster_proxy_connection_establish_error(self): def connect_timeout(): raise requests_exceptions.ConnectTimeout() @@ -221,6 +221,21 @@ class ClusteredAPITestCase(nsxlib_testcase.NsxClientTestCase): self.assertRaises(nsxlib_exc.ServiceClusterUnavailable, api.get, 'api/v1/transport-zones') + def test_cluster_proxy_connection_aborted(self): + + def connect_timeout(): + raise requests_exceptions.ConnectionError("Connection Aborted") + + def all_good(): + pass + + # First call will cause connection aborted error, but next one + # should work + api = self.mock_nsx_clustered_api(session_response=[connect_timeout, + all_good]) + api._validate = mock.Mock() + self.assertEqual(cluster.ClusterHealth.GREEN, api.health) + def test_cluster_round_robin_servicing(self): conf_managers = ['8.9.10.11', '9.10.11.12', '10.11.12.13'] api = self.mock_nsx_clustered_api(nsx_api_managers=conf_managers) diff --git a/vmware_nsxlib/v3/cluster.py b/vmware_nsxlib/v3/cluster.py index 098ae37f..219e46ee 100644 --- a/vmware_nsxlib/v3/cluster.py +++ b/vmware_nsxlib/v3/cluster.py @@ -221,6 +221,9 @@ class NSXRequestsHTTPProvider(AbstractHTTPProvider): def is_connection_exception(self, exception): return isinstance(exception, requests_exceptions.ConnectionError) + def is_conn_open_exception(self, exception): + return isinstance(exception, requests_exceptions.ConnectTimeout) + def get_default_headers(self, session, provider, allow_overwrite_header): """Get the default headers that should be added to future requests""" session.default_headers = {} @@ -584,10 +587,15 @@ class ClusteredAPI(object): if not self._http_provider.is_connection_exception(e): # only trap and retry connection errors raise e - endpoint.set_state(EndpointState.DOWN) + if self._http_provider.is_conn_open_exception(e): + # unable to establish new connection - endpoint is + # inaccessible + endpoint.set_state(EndpointState.DOWN) + LOG.debug("Connection to %s failed, checking additional " - "endpoints" % url) - # retry until exhausting endpoints + "connections and endpoints" % url) + # this might be a result of server closing connection + # retry until exhausting connections and endpoints return self._proxy(proxy_for, uri, *args, **kwargs)