openstack
/
octavia-tempest-plugin
Code Issues Proposed changes

Retry subnet/network deletes on 409 Conflict 

Neutron can be slow to cleanup ports from subnets/networks.
This patch adds retries when deleting subnets and networks in the
tempest teardown/clean up phase after tests.

Also, there were cases where addClassResourceCleanup was being used
inside test cases instead of addCleanup. This patch corrects those to
use addCleanup.

Story: 2004826
Task: 29000

Change-Id: Ia29541d1c89f3559a3ce22b1a27c6bcf079ce2cc
This commit is contained in:
Michael Johnson 2019-01-20 11:03:50 -08:00
parent 0a0f9b342a
commit 04dc5cb4a0
8 changed files with 58 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
octavia_tempest_plugin/tests/scenario/v2/test_healthmonitor.py
View File

@ -103,7 +103,7 @@ class HealthMonitorScenarioTest(test_base.LoadBalancerBaseTest):
}
hm = self.mem_healthmonitor_client.create_healthmonitor(**hm_kwargs)
self.addClassResourceCleanup(
self.addCleanup(
self.mem_healthmonitor_client.cleanup_healthmonitor,
hm[const.ID], lb_client=self.mem_lb_client, lb_id=self.lb_id)

2
octavia_tempest_plugin/tests/scenario/v2/test_l7policy.py
View File

@ -115,7 +115,7 @@ class L7PolicyScenarioTest(test_base.LoadBalancerBaseTest):
}
l7policy = self.mem_l7policy_client.create_l7policy(**l7policy_kwargs)
self.addClassResourceCleanup(
self.addCleanup(
self.mem_l7policy_client.cleanup_l7policy,
l7policy[const.ID],
lb_client=self.mem_lb_client, lb_id=self.lb_id)

2
octavia_tempest_plugin/tests/scenario/v2/test_l7rule.py
View File

@ -112,7 +112,7 @@ class L7RuleScenarioTest(test_base.LoadBalancerBaseTest):
}
l7rule = self.mem_l7rule_client.create_l7rule(**l7rule_kwargs)
self.addClassResourceCleanup(
self.addCleanup(
self.mem_l7rule_client.cleanup_l7rule,
l7rule[const.ID], l7policy_id=self.l7policy_id,
lb_client=self.mem_lb_client, lb_id=self.lb_id)

2
octavia_tempest_plugin/tests/scenario/v2/test_listener.py
View File

@ -132,7 +132,7 @@ class ListenerScenarioTest(test_base.LoadBalancerBaseTest):
})
listener = self.mem_listener_client.create_listener(**listener_kwargs)
self.addClassResourceCleanup(
self.addCleanup(
self.mem_listener_client.cleanup_listener,
listener[const.ID],
lb_client=self.mem_lb_client, lb_id=self.lb_id)

2
octavia_tempest_plugin/tests/scenario/v2/test_load_balancer.py
View File

@ -59,7 +59,7 @@ class LoadBalancerScenarioTest(test_base.LoadBalancerBaseTest):
self._setup_lb_network_kwargs(lb_kwargs, ip_version)
lb = self.mem_lb_client.create_loadbalancer(**lb_kwargs)
self.addClassResourceCleanup(
self.addCleanup(
self.mem_lb_client.cleanup_loadbalancer,
lb[const.ID])

2
octavia_tempest_plugin/tests/scenario/v2/test_member.py
View File

@ -123,7 +123,7 @@ class MemberScenarioTest(test_base.LoadBalancerBaseTest):
const.ID]
member = self.mem_member_client.create_member(**member_kwargs)
self.addClassResourceCleanup(
self.addCleanup(
self.mem_member_client.cleanup_member,
member[const.ID], pool_id=self.pool_id,
lb_client=self.mem_lb_client, lb_id=self.lb_id)

2
octavia_tempest_plugin/tests/scenario/v2/test_pool.py
View File

@ -108,7 +108,7 @@ class PoolScenarioTest(test_base.LoadBalancerBaseTest):
pool_kwargs[const.LOADBALANCER_ID] = self.lb_id
pool = self.mem_pool_client.create_pool(**pool_kwargs)
self.addClassResourceCleanup(
self.addCleanup(
self.mem_pool_client.cleanup_pool,
pool[const.ID],
lb_client=self.mem_lb_client, lb_id=self.lb_id)

60
octavia_tempest_plugin/tests/test_base.py
View File

@ -30,6 +30,7 @@ from tempest.lib.common.utils import data_utils
from tempest.lib.common.utils.linux import remote_client
from tempest.lib import exceptions
from tempest import test
import tenacity
from octavia_tempest_plugin import clients
from octavia_tempest_plugin.common import constants as const
@ -39,6 +40,11 @@ from octavia_tempest_plugin.tests import waiters
CONF = config.CONF
LOG = logging.getLogger(__name__)
RETRY_ATTEMPTS = 15
RETRY_INITIAL_DELAY = 1
RETRY_BACKOFF = 1
RETRY_MAX = 5
class LoadBalancerBaseTest(test.BaseTestCase):
"""Base class for load balancer tests."""
@ -203,6 +209,42 @@ class LoadBalancerBaseTest(test.BaseTestCase):
LOG.debug('Octavia Setup: lb_member_2_ipv6_subnet = {}'.format(
cls.lb_member_2_ipv6_subnet[const.ID]))
@classmethod
# Neutron can be slow to clean up ports from the subnets/networks.
# Retry this delete a few times if we get a "Conflict" error to give
# neutron time to fully cleanup the ports.
@tenacity.retry(
retry=tenacity.retry_if_exception_type(exceptions.Conflict),
wait=tenacity.wait_incrementing(
RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
def _logging_delete_network(cls, net_id):
try:
cls.lb_mem_net_client.delete_network(net_id)
except Exception:
LOG.error('Unable to delete network {}. Active ports:'.format(
net_id))
LOG.error(cls.lb_mem_ports_client.list_ports())
raise
@classmethod
# Neutron can be slow to clean up ports from the subnets/networks.
# Retry this delete a few times if we get a "Conflict" error to give
# neutron time to fully cleanup the ports.
@tenacity.retry(
retry=tenacity.retry_if_exception_type(exceptions.Conflict),
wait=tenacity.wait_incrementing(
RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
def _logging_delete_subnet(cls, subnet_id):
try:
cls.lb_mem_subnet_client.delete_subnet(subnet_id)
except Exception:
LOG.error('Unable to delete subnet {}. Active ports:'.format(
subnet_id))
LOG.error(cls.lb_mem_ports_client.list_ports())
raise
@classmethod
def _create_networks(cls):
"""Creates networks, subnets, and routers used in tests.
@ -230,7 +272,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
LOG.info('lb_member_vip_net: {}'.format(cls.lb_member_vip_net))
cls.addClassResourceCleanup(
waiters.wait_for_not_found,
cls.lb_mem_net_client.delete_network,
cls._logging_delete_network,
cls.lb_mem_net_client.show_network,
cls.lb_member_vip_net['id'])
@ -245,7 +287,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
LOG.info('lb_member_vip_subnet: {}'.format(cls.lb_member_vip_subnet))
cls.addClassResourceCleanup(
waiters.wait_for_not_found,
cls.lb_mem_subnet_client.delete_subnet,
cls._logging_delete_subnet,
cls.lb_mem_subnet_client.show_subnet,
cls.lb_member_vip_subnet['id'])
@ -270,7 +312,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
cls.lb_member_vip_ipv6_subnet = result['subnet']
cls.addClassResourceCleanup(
waiters.wait_for_not_found,
cls.lb_mem_subnet_client.delete_subnet,
cls._logging_delete_subnet,
cls.lb_mem_subnet_client.show_subnet,
cls.lb_member_vip_ipv6_subnet['id'])
LOG.info('lb_member_vip_ipv6_subnet: {}'.format(
@ -289,7 +331,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
LOG.info('lb_member_1_net: {}'.format(cls.lb_member_1_net))
cls.addClassResourceCleanup(
waiters.wait_for_not_found,
cls.lb_mem_net_client.delete_network,
cls._logging_delete_network,
cls.lb_mem_net_client.show_network,
cls.lb_member_1_net['id'])
@ -304,7 +346,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
LOG.info('lb_member_1_subnet: {}'.format(cls.lb_member_1_subnet))
cls.addClassResourceCleanup(
waiters.wait_for_not_found,
cls.lb_mem_subnet_client.delete_subnet,
cls._logging_delete_subnet,
cls.lb_mem_subnet_client.show_subnet,
cls.lb_member_1_subnet['id'])
@ -325,7 +367,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
cls.lb_member_1_ipv6_subnet))
cls.addClassResourceCleanup(
waiters.wait_for_not_found,
cls.lb_mem_subnet_client.delete_subnet,
cls._logging_delete_subnet,
cls.lb_mem_subnet_client.show_subnet,
cls.lb_member_1_ipv6_subnet['id'])
@ -342,7 +384,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
LOG.info('lb_member_2_net: {}'.format(cls.lb_member_2_net))
cls.addClassResourceCleanup(
waiters.wait_for_not_found,
cls.lb_mem_net_client.delete_network,
cls._logging_delete_network,
cls.lb_mem_net_client.show_network,
cls.lb_member_2_net['id'])
@ -357,7 +399,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
LOG.info('lb_member_2_subnet: {}'.format(cls.lb_member_2_subnet))
cls.addClassResourceCleanup(
waiters.wait_for_not_found,
cls.lb_mem_subnet_client.delete_subnet,
cls._logging_delete_subnet,
cls.lb_mem_subnet_client.show_subnet,
cls.lb_member_2_subnet['id'])
@ -378,7 +420,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
cls.lb_member_2_ipv6_subnet))
cls.addClassResourceCleanup(
waiters.wait_for_not_found,
cls.lb_mem_subnet_client.delete_subnet,
cls._logging_delete_subnet,
cls.lb_mem_subnet_client.show_subnet,
cls.lb_member_2_ipv6_subnet['id'])