From efbafb9ea733ea21f2b26471f29867d0636a684b Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Thu, 29 Oct 2020 12:48:08 +0900 Subject: [PATCH] Introduce FT keystone client initialization retry I noticed that keystone client initialization is likely to fail due to a temporary InternalServerError, leading to nondeterministic FT failures. Let's make it tolerable against them. Change-Id: I788645d7d489504b4a1c77da010fe0221bf746b7 Related-Bug: #1886213 --- tacker/tests/functional/keystone.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tacker/tests/functional/keystone.py b/tacker/tests/functional/keystone.py index 3be373f4f..6a48e82de 100644 --- a/tacker/tests/functional/keystone.py +++ b/tacker/tests/functional/keystone.py @@ -14,6 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. +import time + from keystoneauth1 import exceptions from keystoneauth1.identity import v3 from keystoneauth1 import session @@ -24,6 +26,8 @@ from oslo_log import log as logging LOG = logging.getLogger(__name__) CONF = cfg.CONF +KEYSTONE_CLIENT_RETRIES = 3 +KEYSTONE_RETRY_WAIT = 5 class Keystone(object): @@ -51,5 +55,17 @@ class Keystone(object): verify = 'True' == kwargs.pop('cert_verify', 'False') auth_plugin = v3.Password(**kwargs) ses = self.get_session(auth_plugin=auth_plugin, verify=verify) - cli = client.Client('v3', session=ses) + + client_retries = KEYSTONE_CLIENT_RETRIES + while client_retries > 0: + try: + cli = client.Client('v3', session=ses) + break + except exceptions.InternalServerError: + LOG.warning("keystone service responds with 500 " + "(InternalServerError).") + client_retries = client_retries - 1 + if client_retries == 0: + raise + time.sleep(KEYSTONE_RETRY_WAIT) return cli