Translate exception that grounds cluster

When endpoint goes down, the user should see same exception as
when the cluster is already down (detected by earlier activity).
For this purpose, translate grounding exception to
ServiceClusterUnavaliable.
In addition, display a warning if amount of retries is less than
amount of endpoints, since in this case not all endpoints will be
probed.

Change-Id: Ib4aa5eb95069b917c989b1f6dcd3535880b5a038

vmware_nsxlib/tests/unit/v3/test_cluster.py

@@ -428,6 +428,24 @@ class ClusteredAPITestCase(nsxlib_testcase.NsxClientTestCase):
         api.get('api/v1/transport-zones')
         self.assertEqual(cluster.ClusterHealth.GREEN, api.health)
 
+    def test_exception_translation_on_ground_error(self):
+
+        def server_error():
+            raise requests_exceptions.ConnectionError()
+
+        conf_managers = ['8.9.10.11', '9.10.11.12', '10.11.12.13']
+        exceptions = config.ExceptionConfig()
+        max_attempts = 4
+        api = self.mock_nsx_clustered_api(
+            nsx_api_managers=conf_managers,
+            max_attempts=max_attempts,
+            session_response=[server_error for i in range(0, max_attempts)])
+        api.nsxlib_config.exception_config = exceptions
+
+        self.assertRaises(nsxlib_exc.ServiceClusterUnavailable,
+                          api.get, 'api/v1/transport-zones')
+        self.assertEqual(cluster.ClusterHealth.RED, api.health)
+
     def test_cluster_proxy_connection_establish_error(self):
 
         def connect_timeout():

vmware_nsxlib/v3/cluster.py

@@ -665,7 +665,21 @@ class ClusteredAPI(object):
 
     def _proxy_stub(self, proxy_for):
         def _call_proxy(url, *args, **kwargs):
-            return self._proxy(proxy_for, url, *args, **kwargs)
+            try:
+                return self._proxy(proxy_for, url, *args, **kwargs)
+            except Exception as ex:
+                # If this was exception that grounded the cluster,
+                # we want to translate this exception to
+                # ServiceClusterUnavailable. This is in order to
+                # provide unified "cluster down" experience for
+                # the client
+                exc_config = self.nsxlib_config.exception_config
+                if exc_config.should_ground_endpoint(ex):
+                    raise exceptions.ServiceClusterUnavailable(
+                        cluster_id=self.cluster_id)
+
+                raise ex
+
         return _call_proxy
 
     def _proxy(self, proxy_for, uri, *args, **kwargs):

vmware_nsxlib/v3/config.py

@@ -206,6 +206,12 @@ class NsxLibConfig(object):
                         " to True, overriding provided configuration")
             self.cluster_unavailable_retry = True
 
+        if len(nsx_api_managers) > self.max_attempts:
+            LOG.warning("max_attempts setting (%d) is lower than amount of"
+                        " endpoints (%d), which means that not all endpoints"
+                        " will be probed in case of retriable error",
+                        self.max_attempts, len(nsx_api_managers))
+
     def extend(self, keepalive_section, validate_connection_method=None,
                url_base=None):
         if keepalive_section or validate_connection_method: