From 594e214b35619a5b93760902aa2d737bf0865cc3 Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Thu, 29 Oct 2020 12:48:14 +0900 Subject: [PATCH] Allow wait_until_vnf_* to withstand temporary backend failures I noticed that sometimes backend mysqld seems to randomly restart on FT environments, leading to nondeterministic FT test failures. Let's make it tolerable against temporary 5XX errors on show_vnf GET requests within the timeout period, so as to avoid nondeterministic and worthless FT failure noises. Plus, as a side improvement, let's make use of tackerclient's "retries" feature. Related-Bug: #1886213 Change-Id: Ib96034324036524784a8b6014dfc2ed57e3508bc --- tacker/tests/functional/base.py | 37 +++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/tacker/tests/functional/base.py b/tacker/tests/functional/base.py index e53d05bc8..90083b9ce 100644 --- a/tacker/tests/functional/base.py +++ b/tacker/tests/functional/base.py @@ -23,6 +23,7 @@ from keystoneclient import adapter from neutronclient.v2_0 import client as neutron_client from novaclient import client as nova_client from oslo_config import cfg +from oslo_log import log as logging from oslo_serialization import jsonutils from tempest.lib import base @@ -32,9 +33,11 @@ from tacker.tests.functional import clients from tacker.tests.utils import read_file from tacker import version +from tackerclient.common import exceptions from tackerclient.v1_0 import client as tacker_client +LOG = logging.getLogger(__name__) CONF = cfg.CONF @@ -125,7 +128,7 @@ class BaseTackerTest(base.BaseTestCase): @classmethod def tackerclient(cls): auth_session = cls.get_auth_session() - return tacker_client.Client(session=auth_session) + return tacker_client.Client(session=auth_session, retries=5) @classmethod def novaclient(cls): @@ -204,11 +207,17 @@ class BaseTackerTest(base.BaseTestCase): def wait_until_vnf_status(self, vnf_id, target_status, timeout, sleep_interval): start_time = int(time.time()) + status = None while True: - vnf_result = self.client.show_vnf(vnf_id) - status = vnf_result['vnf']['status'] - if (status == target_status) or ( - (int(time.time()) - start_time) > timeout): + try: + vnf_result = self.client.show_vnf(vnf_id) + status = vnf_result['vnf']['status'] + if status == target_status: + break + except exceptions.InternalServerError: + pass + + if int(time.time()) - start_time > timeout: break time.sleep(sleep_interval) @@ -233,18 +242,24 @@ class BaseTackerTest(base.BaseTestCase): constants.VNF_CIRROS_CREATE_TIMEOUT, constants.ACTIVE_SLEEP_TIME) - def wait_until_vnf_delete(self, vnf_id, timeout): + def wait_until_vnf_delete(self, vnf_id, timeout, sleep_interval=1): start_time = int(time.time()) while True: + status = None try: vnf_result = self.client.show_vnf(vnf_id) - time.sleep(1) - except Exception: + status = vnf_result['vnf']['status'] + except exceptions.NotFound: return - status = vnf_result['vnf']['status'] - if (status != 'PENDING_DELETE') or (( - int(time.time()) - start_time) > timeout): + except Exception as e: + LOG.error("Failed to get vnf status: %s", str(e)) + + if status is not None and status != 'PENDING_DELETE': raise Exception("Failed with status: %s" % status) + if int(time.time()) - start_time > timeout: + raise Exception("Timeout for deleting vnf %s.", + vnf_id) + time.sleep(sleep_interval) def wait_until_vnf_dead(self, vnf_id, timeout, sleep_interval): self.wait_until_vnf_status(vnf_id, 'DEAD', timeout,