Regenerate token when invalid JWT token is used to connect to NSX

JWT token used to authenticate with NSX can become invalid before
expiration due to VC service account credentials refresh. When this
case happens nsxlib should immediately re-get-token using the latest
creds and refresh request headers.

Change-Id: I1e3415379926f07e7b30eeaf44e9bcc7e2a26e9e

vmware_nsxlib/v3/client.py

@@ -88,7 +88,8 @@ def http_error_to_exception(status_code, error_code):
              '99': exceptions.ClientCertificateNotTrusted,
              '607': exceptions.APITransactionAborted},
         requests.codes.FORBIDDEN:
-            {'98': exceptions.BadXSRFToken},
+            {'98': exceptions.BadXSRFToken,
+             '403': exceptions.InvalidCredentials},
         requests.codes.TOO_MANY_REQUESTS: exceptions.TooManyRequests,
         requests.codes.SERVICE_UNAVAILABLE: exceptions.ServiceUnavailable}
 

vmware_nsxlib/v3/cluster.py

@@ -228,7 +228,6 @@ class NSXRequestsHTTPProvider(AbstractHTTPProvider):
         self.get_default_headers(session, provider,
                                  config.allow_overwrite_header,
                                  config.token_provider)
-
         return session
 
     def get_default_headers(self, session, provider, allow_overwrite_header,
@@ -649,7 +648,7 @@ class ClusteredAPI(object):
                 # slow rate at once per 33 seconds by default.
                 yield EndpointConnection(endpoint, conn, conn_wait, rate_wait)
 
-    def _raise_http_exception_if_needed(self, response):
+    def _raise_http_exception_if_needed(self, response, endpoint):
         # We need to inspect http codes to understand whether
         # this error is relevant for endpoint-level decisions, such
         # as ground endpoint or retry with next endpoint
@@ -658,6 +657,14 @@ class ClusteredAPI(object):
             # This exception is irrelevant for endpoint decisions
             return
 
+        if (self.nsxlib_config.exception_config.should_regenerate(exc) and
+            bool(self.nsxlib_config.token_provider)):
+            # get new jwt token for authentication
+            self.nsxlib_config.token_provider.get_token(refresh=True)
+            # refresh endpoint so that it gets new header with updated token
+            endpoint.regenerate_pool()
+            raise exc
+
         exc_config = self.nsxlib_config.exception_config
         if (exc_config.should_ground_endpoint(exc) or
             exc_config.should_retry(exc)):
@@ -701,7 +708,7 @@ class ClusteredAPI(object):
 
                     # for some status codes, we need to bring the cluster
                     # down or retry API call
-                    self._raise_http_exception_if_needed(response)
+                    self._raise_http_exception_if_needed(response, endpoint)
 
                     return response
                 except Exception as e:

vmware_nsxlib/v3/config.py

@@ -39,6 +39,11 @@ class ExceptionConfig(object):
                            v3_exceptions.CannotConnectToServer,
                            v3_exceptions.ServerBusy]
 
+        # When hit during API call, these exceptions will be retried
+        # after the endpoints are regenerated with up-to-date auth
+        # credentials / tokens
+        self.regenerate_triggers = [v3_exceptions.InvalidCredentials]
+
     def should_ground_endpoint(self, ex):
         for exception in self.ground_triggers:
             if isinstance(ex, exception):
@@ -53,6 +58,13 @@ class ExceptionConfig(object):
 
         return False
 
+    def should_regenerate(self, ex):
+        for exception in self.regenerate_triggers:
+            if isinstance(ex, exception):
+                return True
+
+        return False
+
 
 class NsxLibConfig(object):
     """Class holding all the configuration parameters used by the nsxlib code.

vmware_nsxlib/v3/exceptions.py

@@ -151,6 +151,10 @@ class BadXSRFToken(ManagerError):
     message = _("Bad or expired XSRF token")
 
 
+class InvalidCredentials(ManagerError):
+    message = _("Failed to authenticate with NSX: %(msg)s")
+
+
 class BadJSONWebTokenProviderRequest(NsxLibException):
     message = _("Bad or expired JSON web token request from provider: %(msg)s")