Merge "Add retries when loading keystone data and fetching endpoints"

heat/common/context.py

@@ -12,6 +12,7 @@
 #    under the License.
 
 from keystoneauth1 import access
+from keystoneauth1 import exceptions as ksa_exceptions
 from keystoneauth1.identity import access as access_plugin
 from keystoneauth1.identity import generic
 from keystoneauth1 import loading as ks_loading
@@ -24,6 +25,7 @@ import oslo_messaging
 from oslo_middleware import request_id as oslo_request_id
 from oslo_utils import importutils
 import six
+import tenacity
 
 from heat.common import config
 from heat.common import endpoint_utils
@@ -51,6 +53,15 @@ TRUSTEE_CONF_GROUP = 'trustee'
 ks_loading.register_auth_conf_options(cfg.CONF, TRUSTEE_CONF_GROUP)
 
 
+retry_on_connection_timeout = tenacity.retry(
+    stop=tenacity.stop_after_attempt(cfg.CONF.client_retry_limit+1),
+    wait=tenacity.wait_random(max=2),
+    retry=tenacity.retry_if_exception_type(
+        (ksa_exceptions.ConnectFailure,
+         ksa_exceptions.DiscoveryFailure)),
+    reraise=True)
+
+
 def list_opts():
     trustee_opts = ks_loading.get_auth_common_conf_options()
     trustee_opts.extend(ks_loading.get_auth_plugin_conf_options(
@@ -288,6 +299,8 @@ class RequestContext(context.RequestContext):
 
 
 class StoredContext(RequestContext):
+
+    @retry_on_connection_timeout
     def _load_keystone_data(self):
         self._keystone_loaded = True
         auth_ref = self.auth_plugin.get_access(self.keystone_session)

heat/engine/clients/client_plugin.py

@@ -24,6 +24,7 @@ import requests
 import six
 
 from heat.common import config
+from heat.common import context
 from heat.common import exception as heat_exception
 
 cfg.CONF.import_opt('client_retry_limit', 'heat.common.config')
@@ -93,6 +94,7 @@ class ClientPlugin(object):
     def url_for(self, **kwargs):
         keystone_session = self.context.keystone_session
 
+        @context.retry_on_connection_timeout
         def get_endpoint():
             return keystone_session.get_endpoint(**kwargs)