Retry http_request and json_request failure.

Temporary network outages or keystone server restarts can lead
to a connection refused in auth_token middleware. Rather than
failing immediately, this patch attempts to retry a few times.

Fixes bug 1150299

Change-Id: I2ecf0d7745290976efcb3e3cd6511817a53d3e0a

keystoneclient/middleware/auth_token.py

@@ -559,25 +559,36 @@ class AuthProtocol(object):
                                           self.cert_file,
                                           timeout=self.http_connect_timeout)
 
-    def _http_request(self, method, path):
+    def _http_request(self, method, path, **kwargs):
         """HTTP request helper used to make unspecified content type requests.
 
         :param method: http method
         :param path: relative request url
-        :return (http response object)
+        :return (http response object, response body)
         :raise ServerError when unable to communicate with keystone
 
         """
         conn = self._get_http_connection()
-        try:
+
-            conn.request(method, path)
+        RETRIES = 3
-            response = conn.getresponse()
+        retry = 0
-            body = response.read()
+
-        except Exception as e:
+        while True:
-            self.LOG.error('HTTP connection exception: %s' % e)
+            try:
-            raise ServiceError('Unable to communicate with keystone')
+                conn.request(method, path, **kwargs)
-        finally:
+                response = conn.getresponse()
-            conn.close()
+                body = response.read()
+                break
+            except Exception as e:
+                if retry == RETRIES:
+                    self.LOG.error('HTTP connection exception: %s' % e)
+                    raise ServiceError('Unable to communicate with keystone')
+                # NOTE(vish): sleep 0.5, 1, 2
+                self.LOG.warn('Retrying on HTTP connection exception: %s' % e)
+                time.sleep(2.0 ** retry / 2)
+                retry += 1
+            finally:
+                conn.close()
 
         return response, body
 
@@ -593,8 +604,6 @@ class AuthProtocol(object):
         :raise ServerError when unable to communicate with keystone
 
         """
-        conn = self._get_http_connection()
-
         kwargs = {
             'headers': {
                 'Content-type': 'application/json',
@@ -608,16 +617,10 @@ class AuthProtocol(object):
         if body:
             kwargs['body'] = jsonutils.dumps(body)
 
-        full_path = self.auth_admin_prefix + path
+        path = self.auth_admin_prefix + path
-        try:
+
-            conn.request(method, full_path, **kwargs)
+        response, body = self._http_request(method, path, **kwargs)
-            response = conn.getresponse()
+
-            body = response.read()
-        except Exception as e:
-            self.LOG.error('HTTP connection exception: %s' % e)
-            raise ServiceError('Unable to communicate with keystone')
-        finally:
-            conn.close()
         try:
             data = jsonutils.loads(body)
         except ValueError: