NSX: Add ability to retry on 503's returned by the controller

There are a number of circumstances where the NSX controller
may return 503. Currently the API client does not retry, so
this patch adds a retry logic with timeout.

Closes-bug: #1284277

Change-Id: I85df087d5ae409e6cb5c35eb171e89346abe81f4

neutron/plugins/vmware/api_client/eventlet_request.py

@@ -128,17 +128,21 @@ class EventletApiRequest(request.ApiRequest):
     def _handle_request(self):
         '''First level request handling.'''
         attempt = 0
+        timeout = 0
         response = None
         while response is None and attempt <= self._retries:
-            eventlet.greenthread.sleep(0)
+            eventlet.greenthread.sleep(timeout)
             attempt += 1
 
             req = self._issue_request()
             # automatically raises any exceptions returned.
             if isinstance(req, httplib.HTTPResponse):
+                timeout = 0
                 if attempt <= self._retries and not self._abort:
-                    if (req.status == httplib.UNAUTHORIZED
-                        or req.status == httplib.FORBIDDEN):
+                    if req.status in (httplib.UNAUTHORIZED, httplib.FORBIDDEN):
+                        continue
+                    elif req.status == httplib.SERVICE_UNAVAILABLE:
+                        timeout = 0.5
                         continue
                     # else fall through to return the error code