Merge "Handle cluster connection closed by server"

vmware_nsxlib/tests/unit/v3/test_cluster.py

@@ -211,7 +211,7 @@ class ClusteredAPITestCase(nsxlib_testcase.NsxClientTestCase):
         self.assertRaises(nsxlib_exc.StaleRevision,
                           api.get, 'api/v1/transport-zones')
 
-    def test_cluster_proxy_connection_error(self):
+    def test_cluster_proxy_connection_establish_error(self):
 
         def connect_timeout():
             raise requests_exceptions.ConnectTimeout()
@@ -221,6 +221,21 @@ class ClusteredAPITestCase(nsxlib_testcase.NsxClientTestCase):
         self.assertRaises(nsxlib_exc.ServiceClusterUnavailable,
                           api.get, 'api/v1/transport-zones')
 
+    def test_cluster_proxy_connection_aborted(self):
+
+        def connect_timeout():
+            raise requests_exceptions.ConnectionError("Connection Aborted")
+
+        def all_good():
+            pass
+
+        # First call will cause connection aborted error, but next one
+        # should work
+        api = self.mock_nsx_clustered_api(session_response=[connect_timeout,
+                                                            all_good])
+        api._validate = mock.Mock()
+        self.assertEqual(cluster.ClusterHealth.GREEN, api.health)
+
     def test_cluster_round_robin_servicing(self):
         conf_managers = ['8.9.10.11', '9.10.11.12', '10.11.12.13']
         api = self.mock_nsx_clustered_api(nsx_api_managers=conf_managers)

vmware_nsxlib/v3/cluster.py

@@ -221,6 +221,9 @@ class NSXRequestsHTTPProvider(AbstractHTTPProvider):
     def is_connection_exception(self, exception):
         return isinstance(exception, requests_exceptions.ConnectionError)
 
+    def is_conn_open_exception(self, exception):
+        return isinstance(exception, requests_exceptions.ConnectTimeout)
+
     def get_default_headers(self, session, provider, allow_overwrite_header):
         """Get the default headers that should be added to future requests"""
         session.default_headers = {}
@@ -584,10 +587,15 @@ class ClusteredAPI(object):
                 if not self._http_provider.is_connection_exception(e):
                     # only trap and retry connection errors
                     raise e
-                endpoint.set_state(EndpointState.DOWN)
+                if self._http_provider.is_conn_open_exception(e):
+                    # unable to establish new connection - endpoint is
+                    # inaccessible
+                    endpoint.set_state(EndpointState.DOWN)
+
                 LOG.debug("Connection to %s failed, checking additional "
-                          "endpoints" % url)
-                # retry until exhausting endpoints
+                          "connections and endpoints" % url)
+                # this might be a result of server closing connection
+                # retry until exhausting connections and endpoints
                 return self._proxy(proxy_for, uri, *args, **kwargs)