Browse Source

Add retries when loading keystone data and fetching endpoints

We may end up loading lots of nested stacks concurrently with
convergence and those would try to discover endpoints and fetch
access info objects from keystone. This at times results in
ConnectTimeout errors from keystone. We can avoid these errors
by adding some retries.

Also adds retries to client_plugin get_endpoint() calls, which has
simillar issue.

Change-Id: I18cde971248eff5783f97c9e7a60316d7dd93431
Task: 36349
(cherry picked from commit 6fb8ac250a)
changes/41/678341/1
Rabi Mishra 3 weeks ago
parent
commit
302add5bb8
2 changed files with 15 additions and 0 deletions
  1. 13
    0
      heat/common/context.py
  2. 2
    0
      heat/engine/clients/client_plugin.py

+ 13
- 0
heat/common/context.py View File

@@ -12,6 +12,7 @@
12 12
 #    under the License.
13 13
 
14 14
 from keystoneauth1 import access
15
+from keystoneauth1 import exceptions as ksa_exceptions
15 16
 from keystoneauth1.identity import access as access_plugin
16 17
 from keystoneauth1.identity import generic
17 18
 from keystoneauth1 import loading as ks_loading
@@ -24,6 +25,7 @@ import oslo_messaging
24 25
 from oslo_middleware import request_id as oslo_request_id
25 26
 from oslo_utils import importutils
26 27
 import six
28
+import tenacity
27 29
 
28 30
 from heat.common import config
29 31
 from heat.common import endpoint_utils
@@ -51,6 +53,15 @@ TRUSTEE_CONF_GROUP = 'trustee'
51 53
 ks_loading.register_auth_conf_options(cfg.CONF, TRUSTEE_CONF_GROUP)
52 54
 
53 55
 
56
+retry_on_connection_timeout = tenacity.retry(
57
+    stop=tenacity.stop_after_attempt(cfg.CONF.client_retry_limit+1),
58
+    wait=tenacity.wait_random(max=2),
59
+    retry=tenacity.retry_if_exception_type(
60
+        (ksa_exceptions.ConnectFailure,
61
+         ksa_exceptions.DiscoveryFailure)),
62
+    reraise=True)
63
+
64
+
54 65
 def list_opts():
55 66
     trustee_opts = ks_loading.get_auth_common_conf_options()
56 67
     trustee_opts.extend(ks_loading.get_auth_plugin_conf_options(
@@ -288,6 +299,8 @@ class RequestContext(context.RequestContext):
288 299
 
289 300
 
290 301
 class StoredContext(RequestContext):
302
+
303
+    @retry_on_connection_timeout
291 304
     def _load_keystone_data(self):
292 305
         self._keystone_loaded = True
293 306
         auth_ref = self.auth_plugin.get_access(self.keystone_session)

+ 2
- 0
heat/engine/clients/client_plugin.py View File

@@ -24,6 +24,7 @@ import requests
24 24
 import six
25 25
 
26 26
 from heat.common import config
27
+from heat.common import context
27 28
 from heat.common import exception as heat_exception
28 29
 
29 30
 cfg.CONF.import_opt('client_retry_limit', 'heat.common.config')
@@ -93,6 +94,7 @@ class ClientPlugin(object):
93 94
     def url_for(self, **kwargs):
94 95
         keystone_session = self.context.keystone_session
95 96
 
97
+        @context.retry_on_connection_timeout
96 98
         def get_endpoint():
97 99
             return keystone_session.get_endpoint(**kwargs)
98 100
 

Loading…
Cancel
Save