Allow wait_until_vnf_* to withstand temporary backend failures

I noticed that sometimes backend mysqld seems to randomly restart on FT
environments, leading to nondeterministic FT test failures.
Let's make it tolerable against temporary 5XX errors on show_vnf GET
requests within the timeout period, so as to avoid nondeterministic
and worthless FT failure noises.

Plus, as a side improvement, let's make use of tackerclient's "retries"
feature.

Related-Bug: #1886213
Change-Id: Ib96034324036524784a8b6014dfc2ed57e3508bc
This commit is contained in:
Koichiro Den 2020-10-29 12:48:14 +09:00
parent 25f93312c1
commit 594e214b35
1 changed files with 26 additions and 11 deletions

View File

@ -23,6 +23,7 @@ from keystoneclient import adapter
from neutronclient.v2_0 import client as neutron_client
from novaclient import client as nova_client
from oslo_config import cfg
from oslo_log import log as logging
from oslo_serialization import jsonutils
from tempest.lib import base
@ -32,9 +33,11 @@ from tacker.tests.functional import clients
from tacker.tests.utils import read_file
from tacker import version
from tackerclient.common import exceptions
from tackerclient.v1_0 import client as tacker_client
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
@ -125,7 +128,7 @@ class BaseTackerTest(base.BaseTestCase):
@classmethod
def tackerclient(cls):
auth_session = cls.get_auth_session()
return tacker_client.Client(session=auth_session)
return tacker_client.Client(session=auth_session, retries=5)
@classmethod
def novaclient(cls):
@ -204,11 +207,17 @@ class BaseTackerTest(base.BaseTestCase):
def wait_until_vnf_status(self, vnf_id, target_status, timeout,
sleep_interval):
start_time = int(time.time())
status = None
while True:
vnf_result = self.client.show_vnf(vnf_id)
status = vnf_result['vnf']['status']
if (status == target_status) or (
(int(time.time()) - start_time) > timeout):
try:
vnf_result = self.client.show_vnf(vnf_id)
status = vnf_result['vnf']['status']
if status == target_status:
break
except exceptions.InternalServerError:
pass
if int(time.time()) - start_time > timeout:
break
time.sleep(sleep_interval)
@ -233,18 +242,24 @@ class BaseTackerTest(base.BaseTestCase):
constants.VNF_CIRROS_CREATE_TIMEOUT,
constants.ACTIVE_SLEEP_TIME)
def wait_until_vnf_delete(self, vnf_id, timeout):
def wait_until_vnf_delete(self, vnf_id, timeout, sleep_interval=1):
start_time = int(time.time())
while True:
status = None
try:
vnf_result = self.client.show_vnf(vnf_id)
time.sleep(1)
except Exception:
status = vnf_result['vnf']['status']
except exceptions.NotFound:
return
status = vnf_result['vnf']['status']
if (status != 'PENDING_DELETE') or ((
int(time.time()) - start_time) > timeout):
except Exception as e:
LOG.error("Failed to get vnf status: %s", str(e))
if status is not None and status != 'PENDING_DELETE':
raise Exception("Failed with status: %s" % status)
if int(time.time()) - start_time > timeout:
raise Exception("Timeout for deleting vnf %s.",
vnf_id)
time.sleep(sleep_interval)
def wait_until_vnf_dead(self, vnf_id, timeout, sleep_interval):
self.wait_until_vnf_status(vnf_id, 'DEAD', timeout,