openstack
/
heat
Code Issues Proposed changes

Add retries when loading keystone data and fetching endpoints 

We may end up loading lots of nested stacks concurrently with
convergence and those would try to discover endpoints and fetch
access info objects from keystone. This at times results in
ConnectTimeout errors from keystone. We can avoid these errors
by adding some retries.

Also adds retries to client_plugin get_endpoint() calls, which has
simillar issue.

Change-Id: I18cde971248eff5783f97c9e7a60316d7dd93431
Task: 36349
This commit is contained in:
Rabi Mishra 2019-08-23 10:45:28 +05:30
parent 364716725a
commit 6fb8ac250a
2 changed files with 15 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
heat/common/context.py
View File

@ -12,6 +12,7 @@
# under the License. # under the License.
from keystoneauth1 import access from keystoneauth1 import access
from keystoneauth1 import exceptions as ksa_exceptions
from keystoneauth1.identity import access as access_plugin from keystoneauth1.identity import access as access_plugin
from keystoneauth1.identity import generic from keystoneauth1.identity import generic
from keystoneauth1 import loading as ks_loading from keystoneauth1 import loading as ks_loading
@ -24,6 +25,7 @@ import oslo_messaging
from oslo_middleware import request_id as oslo_request_id from oslo_middleware import request_id as oslo_request_id
from oslo_utils import importutils from oslo_utils import importutils
import six import six
import tenacity
from heat.common import config from heat.common import config
from heat.common import endpoint_utils from heat.common import endpoint_utils
@ -51,6 +53,15 @@ TRUSTEE_CONF_GROUP = 'trustee'
ks_loading.register_auth_conf_options(cfg.CONF, TRUSTEE_CONF_GROUP) ks_loading.register_auth_conf_options(cfg.CONF, TRUSTEE_CONF_GROUP)
retry_on_connection_timeout = tenacity.retry(
stop=tenacity.stop_after_attempt(cfg.CONF.client_retry_limit+1),
wait=tenacity.wait_random(max=2),
retry=tenacity.retry_if_exception_type(
(ksa_exceptions.ConnectFailure,
ksa_exceptions.DiscoveryFailure)),
reraise=True)
def list_opts(): def list_opts():
trustee_opts = ks_loading.get_auth_common_conf_options() trustee_opts = ks_loading.get_auth_common_conf_options()
trustee_opts.extend(ks_loading.get_auth_plugin_conf_options( trustee_opts.extend(ks_loading.get_auth_plugin_conf_options(
@ -288,6 +299,8 @@ class RequestContext(context.RequestContext):
class StoredContext(RequestContext): class StoredContext(RequestContext):
@retry_on_connection_timeout
def _load_keystone_data(self): def _load_keystone_data(self):
self._keystone_loaded = True self._keystone_loaded = True
auth_ref = self.auth_plugin.get_access(self.keystone_session) auth_ref = self.auth_plugin.get_access(self.keystone_session)

2
heat/engine/clients/client_plugin.py
View File

@ -24,6 +24,7 @@ import requests
import six import six
from heat.common import config from heat.common import config
from heat.common import context
from heat.common import exception as heat_exception from heat.common import exception as heat_exception
cfg.CONF.import_opt('client_retry_limit', 'heat.common.config') cfg.CONF.import_opt('client_retry_limit', 'heat.common.config')
@ -93,6 +94,7 @@ class ClientPlugin(object):
def url_for(self, **kwargs): def url_for(self, **kwargs):
keystone_session = self.context.keystone_session keystone_session = self.context.keystone_session
@context.retry_on_connection_timeout
def get_endpoint(): def get_endpoint():
return keystone_session.get_endpoint(**kwargs) return keystone_session.get_endpoint(**kwargs)