From dd8e5614519c19959e19bfdcea52f437c62a67c5 Mon Sep 17 00:00:00 2001
From: Adit Sarfaty <asarfaty@vmware.com>
Date: Sun, 27 May 2018 12:25:59 +0300
Subject: [PATCH] Retry on 503 Service Unavailable

When the NSX handles too many active requests, it may return HTTP
response 503.
We should handle it the same as 429 (Too many requests), and retry
the request.

Change-Id: I27c1142f5d896ce88c2d5aa70536ad0876efbcfa
---
 vmware_nsxlib/v3/client.py     |  8 +++++---
 vmware_nsxlib/v3/exceptions.py | 10 +++++++++-
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/vmware_nsxlib/v3/client.py b/vmware_nsxlib/v3/client.py
index fb5d5339..3567da7e 100644
--- a/vmware_nsxlib/v3/client.py
+++ b/vmware_nsxlib/v3/client.py
@@ -43,7 +43,8 @@ def http_error_to_exception(status_code, error_code):
             {'99': exceptions.ClientCertificateNotTrusted},
         requests.codes.FORBIDDEN:
             {'98': exceptions.BadXSRFToken},
-        requests.codes.TOO_MANY_REQUESTS: exceptions.TooManyRequests}
+        requests.codes.TOO_MANY_REQUESTS: exceptions.TooManyRequests,
+        requests.codes.SERVICE_UNAVAILABLE: exceptions.ServiceUnavailable}
 
     if status_code in errors:
         if isinstance(errors[status_code], dict):
@@ -306,11 +307,12 @@ class NSX3Client(JSONRESTClient):
     def _rest_call(self, url, **kwargs):
         if self.rate_limit_retry:
             # If too many requests are handled by the nsx at the same time,
-            # error "429: Too Many Requests" will be returned.
+            # error "429: Too Many Requests" or "503: Server Unavailable"
+            # will be returned.
             # the client is expected to retry after a random 400-600 milli,
             # and later exponentially until 5 seconds wait
             @utils.retry_random_upon_exception(
-                exceptions.TooManyRequests,
+                exceptions.ServerBusy,
                 max_attempts=self.max_attempts)
             def _rest_call_with_retry(self, url, **kwargs):
                 return super(NSX3Client, self)._rest_call(url, **kwargs)
diff --git a/vmware_nsxlib/v3/exceptions.py b/vmware_nsxlib/v3/exceptions.py
index 749c8ebd..31d7bf42 100644
--- a/vmware_nsxlib/v3/exceptions.py
+++ b/vmware_nsxlib/v3/exceptions.py
@@ -101,7 +101,15 @@ class StaleRevision(ManagerError):
     pass
 
 
-class TooManyRequests(ManagerError):
+class ServerBusy(ManagerError):
+    pass
+
+
+class TooManyRequests(ServerBusy):
+    pass
+
+
+class ServiceUnavailable(ServerBusy):
     pass