Handle cluster connection closed by server

When established connection is closed by server, endpoint should
not go down, and request should be retried with another connection.
However, if connection fails to be established, endpoint should go
down as coded before.

We observe server closing connections in spite of keep-alive with
policy endpoint.

Change-Id: I264da0ad47c31c9875a4be35acd4c5c4c88f4916

vmware_nsxlib/tests/unit/v3/test_cluster.py

@@ -211,7 +211,7 @@ class ClusteredAPITestCase(nsxlib_testcase.NsxClientTestCase):
         self.assertRaises(nsxlib_exc.StaleRevision,
                           api.get, 'api/v1/transport-zones')
 
-    def test_cluster_proxy_connection_error(self):
+    def test_cluster_proxy_connection_establish_error(self):
 
         def connect_timeout():
             raise requests_exceptions.ConnectTimeout()
@@ -221,6 +221,21 @@ class ClusteredAPITestCase(nsxlib_testcase.NsxClientTestCase):
         self.assertRaises(nsxlib_exc.ServiceClusterUnavailable,
                           api.get, 'api/v1/transport-zones')
 
+    def test_cluster_proxy_connection_aborted(self):
+
+        def connect_timeout():
+            raise requests_exceptions.ConnectionError("Connection Aborted")
+
+        def all_good():
+            pass
+
+        # First call will cause connection aborted error, but next one
+        # should work
+        api = self.mock_nsx_clustered_api(session_response=[connect_timeout,
+                                                            all_good])
+        api._validate = mock.Mock()
+        self.assertEqual(cluster.ClusterHealth.GREEN, api.health)
+
     def test_cluster_round_robin_servicing(self):
         conf_managers = ['8.9.10.11', '9.10.11.12', '10.11.12.13']
         api = self.mock_nsx_clustered_api(nsx_api_managers=conf_managers)

vmware_nsxlib/v3/cluster.py

@@ -221,6 +221,9 @@ class NSXRequestsHTTPProvider(AbstractHTTPProvider):
     def is_connection_exception(self, exception):
         return isinstance(exception, requests_exceptions.ConnectionError)
 
+    def is_conn_open_exception(self, exception):
+        return isinstance(exception, requests_exceptions.ConnectTimeout)
+
     def get_default_headers(self, session, provider, allow_overwrite_header):
         """Get the default headers that should be added to future requests"""
         session.default_headers = {}
@@ -584,10 +587,15 @@ class ClusteredAPI(object):
                 if not self._http_provider.is_connection_exception(e):
                     # only trap and retry connection errors
                     raise e
-                endpoint.set_state(EndpointState.DOWN)
+                if self._http_provider.is_conn_open_exception(e):
+                    # unable to establish new connection - endpoint is
+                    # inaccessible
+                    endpoint.set_state(EndpointState.DOWN)
+
                 LOG.debug("Connection to %s failed, checking additional "
-                          "endpoints" % url)
-                # retry until exhausting endpoints
+                          "connections and endpoints" % url)
+                # this might be a result of server closing connection
+                # retry until exhausting connections and endpoints
                 return self._proxy(proxy_for, uri, *args, **kwargs)