Browse Source

Retry subnet/network deletes on 409 Conflict

Neutron can be slow to cleanup ports from subnets/networks.
This patch adds retries when deleting subnets and networks in the
tempest teardown/clean up phase after tests.

Also, there were cases where addClassResourceCleanup was being used
inside test cases instead of addCleanup. This patch corrects those to
use addCleanup.

Story: 2004826
Task: 29000

Change-Id: Ia29541d1c89f3559a3ce22b1a27c6bcf079ce2cc
Michael Johnson 3 months ago
parent
commit
04dc5cb4a0

+ 1
- 1
octavia_tempest_plugin/tests/scenario/v2/test_healthmonitor.py View File

@@ -103,7 +103,7 @@ class HealthMonitorScenarioTest(test_base.LoadBalancerBaseTest):
103 103
         }
104 104
 
105 105
         hm = self.mem_healthmonitor_client.create_healthmonitor(**hm_kwargs)
106
-        self.addClassResourceCleanup(
106
+        self.addCleanup(
107 107
             self.mem_healthmonitor_client.cleanup_healthmonitor,
108 108
             hm[const.ID], lb_client=self.mem_lb_client, lb_id=self.lb_id)
109 109
 

+ 1
- 1
octavia_tempest_plugin/tests/scenario/v2/test_l7policy.py View File

@@ -115,7 +115,7 @@ class L7PolicyScenarioTest(test_base.LoadBalancerBaseTest):
115 115
         }
116 116
 
117 117
         l7policy = self.mem_l7policy_client.create_l7policy(**l7policy_kwargs)
118
-        self.addClassResourceCleanup(
118
+        self.addCleanup(
119 119
             self.mem_l7policy_client.cleanup_l7policy,
120 120
             l7policy[const.ID],
121 121
             lb_client=self.mem_lb_client, lb_id=self.lb_id)

+ 1
- 1
octavia_tempest_plugin/tests/scenario/v2/test_l7rule.py View File

@@ -112,7 +112,7 @@ class L7RuleScenarioTest(test_base.LoadBalancerBaseTest):
112 112
         }
113 113
 
114 114
         l7rule = self.mem_l7rule_client.create_l7rule(**l7rule_kwargs)
115
-        self.addClassResourceCleanup(
115
+        self.addCleanup(
116 116
             self.mem_l7rule_client.cleanup_l7rule,
117 117
             l7rule[const.ID], l7policy_id=self.l7policy_id,
118 118
             lb_client=self.mem_lb_client, lb_id=self.lb_id)

+ 1
- 1
octavia_tempest_plugin/tests/scenario/v2/test_listener.py View File

@@ -132,7 +132,7 @@ class ListenerScenarioTest(test_base.LoadBalancerBaseTest):
132 132
             })
133 133
 
134 134
         listener = self.mem_listener_client.create_listener(**listener_kwargs)
135
-        self.addClassResourceCleanup(
135
+        self.addCleanup(
136 136
             self.mem_listener_client.cleanup_listener,
137 137
             listener[const.ID],
138 138
             lb_client=self.mem_lb_client, lb_id=self.lb_id)

+ 1
- 1
octavia_tempest_plugin/tests/scenario/v2/test_load_balancer.py View File

@@ -59,7 +59,7 @@ class LoadBalancerScenarioTest(test_base.LoadBalancerBaseTest):
59 59
         self._setup_lb_network_kwargs(lb_kwargs, ip_version)
60 60
 
61 61
         lb = self.mem_lb_client.create_loadbalancer(**lb_kwargs)
62
-        self.addClassResourceCleanup(
62
+        self.addCleanup(
63 63
             self.mem_lb_client.cleanup_loadbalancer,
64 64
             lb[const.ID])
65 65
 

+ 1
- 1
octavia_tempest_plugin/tests/scenario/v2/test_member.py View File

@@ -123,7 +123,7 @@ class MemberScenarioTest(test_base.LoadBalancerBaseTest):
123 123
                 const.ID]
124 124
 
125 125
         member = self.mem_member_client.create_member(**member_kwargs)
126
-        self.addClassResourceCleanup(
126
+        self.addCleanup(
127 127
             self.mem_member_client.cleanup_member,
128 128
             member[const.ID], pool_id=self.pool_id,
129 129
             lb_client=self.mem_lb_client, lb_id=self.lb_id)

+ 1
- 1
octavia_tempest_plugin/tests/scenario/v2/test_pool.py View File

@@ -108,7 +108,7 @@ class PoolScenarioTest(test_base.LoadBalancerBaseTest):
108 108
             pool_kwargs[const.LOADBALANCER_ID] = self.lb_id
109 109
 
110 110
         pool = self.mem_pool_client.create_pool(**pool_kwargs)
111
-        self.addClassResourceCleanup(
111
+        self.addCleanup(
112 112
             self.mem_pool_client.cleanup_pool,
113 113
             pool[const.ID],
114 114
             lb_client=self.mem_lb_client, lb_id=self.lb_id)

+ 51
- 9
octavia_tempest_plugin/tests/test_base.py View File

@@ -30,6 +30,7 @@ from tempest.lib.common.utils import data_utils
30 30
 from tempest.lib.common.utils.linux import remote_client
31 31
 from tempest.lib import exceptions
32 32
 from tempest import test
33
+import tenacity
33 34
 
34 35
 from octavia_tempest_plugin import clients
35 36
 from octavia_tempest_plugin.common import constants as const
@@ -39,6 +40,11 @@ from octavia_tempest_plugin.tests import waiters
39 40
 CONF = config.CONF
40 41
 LOG = logging.getLogger(__name__)
41 42
 
43
+RETRY_ATTEMPTS = 15
44
+RETRY_INITIAL_DELAY = 1
45
+RETRY_BACKOFF = 1
46
+RETRY_MAX = 5
47
+
42 48
 
43 49
 class LoadBalancerBaseTest(test.BaseTestCase):
44 50
     """Base class for load balancer tests."""
@@ -203,6 +209,42 @@ class LoadBalancerBaseTest(test.BaseTestCase):
203 209
                 LOG.debug('Octavia Setup: lb_member_2_ipv6_subnet = {}'.format(
204 210
                     cls.lb_member_2_ipv6_subnet[const.ID]))
205 211
 
212
+    @classmethod
213
+    # Neutron can be slow to clean up ports from the subnets/networks.
214
+    # Retry this delete a few times if we get a "Conflict" error to give
215
+    # neutron time to fully cleanup the ports.
216
+    @tenacity.retry(
217
+        retry=tenacity.retry_if_exception_type(exceptions.Conflict),
218
+        wait=tenacity.wait_incrementing(
219
+            RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
220
+        stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
221
+    def _logging_delete_network(cls, net_id):
222
+        try:
223
+            cls.lb_mem_net_client.delete_network(net_id)
224
+        except Exception:
225
+            LOG.error('Unable to delete network {}. Active ports:'.format(
226
+                net_id))
227
+            LOG.error(cls.lb_mem_ports_client.list_ports())
228
+            raise
229
+
230
+    @classmethod
231
+    # Neutron can be slow to clean up ports from the subnets/networks.
232
+    # Retry this delete a few times if we get a "Conflict" error to give
233
+    # neutron time to fully cleanup the ports.
234
+    @tenacity.retry(
235
+        retry=tenacity.retry_if_exception_type(exceptions.Conflict),
236
+        wait=tenacity.wait_incrementing(
237
+            RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
238
+        stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
239
+    def _logging_delete_subnet(cls, subnet_id):
240
+        try:
241
+            cls.lb_mem_subnet_client.delete_subnet(subnet_id)
242
+        except Exception:
243
+            LOG.error('Unable to delete subnet {}. Active ports:'.format(
244
+                subnet_id))
245
+            LOG.error(cls.lb_mem_ports_client.list_ports())
246
+            raise
247
+
206 248
     @classmethod
207 249
     def _create_networks(cls):
208 250
         """Creates networks, subnets, and routers used in tests.
@@ -230,7 +272,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
230 272
         LOG.info('lb_member_vip_net: {}'.format(cls.lb_member_vip_net))
231 273
         cls.addClassResourceCleanup(
232 274
             waiters.wait_for_not_found,
233
-            cls.lb_mem_net_client.delete_network,
275
+            cls._logging_delete_network,
234 276
             cls.lb_mem_net_client.show_network,
235 277
             cls.lb_member_vip_net['id'])
236 278
 
@@ -245,7 +287,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
245 287
         LOG.info('lb_member_vip_subnet: {}'.format(cls.lb_member_vip_subnet))
246 288
         cls.addClassResourceCleanup(
247 289
             waiters.wait_for_not_found,
248
-            cls.lb_mem_subnet_client.delete_subnet,
290
+            cls._logging_delete_subnet,
249 291
             cls.lb_mem_subnet_client.show_subnet,
250 292
             cls.lb_member_vip_subnet['id'])
251 293
 
@@ -270,7 +312,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
270 312
                 cls.lb_member_vip_ipv6_subnet = result['subnet']
271 313
                 cls.addClassResourceCleanup(
272 314
                     waiters.wait_for_not_found,
273
-                    cls.lb_mem_subnet_client.delete_subnet,
315
+                    cls._logging_delete_subnet,
274 316
                     cls.lb_mem_subnet_client.show_subnet,
275 317
                     cls.lb_member_vip_ipv6_subnet['id'])
276 318
             LOG.info('lb_member_vip_ipv6_subnet: {}'.format(
@@ -289,7 +331,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
289 331
         LOG.info('lb_member_1_net: {}'.format(cls.lb_member_1_net))
290 332
         cls.addClassResourceCleanup(
291 333
             waiters.wait_for_not_found,
292
-            cls.lb_mem_net_client.delete_network,
334
+            cls._logging_delete_network,
293 335
             cls.lb_mem_net_client.show_network,
294 336
             cls.lb_member_1_net['id'])
295 337
 
@@ -304,7 +346,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
304 346
         LOG.info('lb_member_1_subnet: {}'.format(cls.lb_member_1_subnet))
305 347
         cls.addClassResourceCleanup(
306 348
             waiters.wait_for_not_found,
307
-            cls.lb_mem_subnet_client.delete_subnet,
349
+            cls._logging_delete_subnet,
308 350
             cls.lb_mem_subnet_client.show_subnet,
309 351
             cls.lb_member_1_subnet['id'])
310 352
 
@@ -325,7 +367,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
325 367
                 cls.lb_member_1_ipv6_subnet))
326 368
             cls.addClassResourceCleanup(
327 369
                 waiters.wait_for_not_found,
328
-                cls.lb_mem_subnet_client.delete_subnet,
370
+                cls._logging_delete_subnet,
329 371
                 cls.lb_mem_subnet_client.show_subnet,
330 372
                 cls.lb_member_1_ipv6_subnet['id'])
331 373
 
@@ -342,7 +384,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
342 384
         LOG.info('lb_member_2_net: {}'.format(cls.lb_member_2_net))
343 385
         cls.addClassResourceCleanup(
344 386
             waiters.wait_for_not_found,
345
-            cls.lb_mem_net_client.delete_network,
387
+            cls._logging_delete_network,
346 388
             cls.lb_mem_net_client.show_network,
347 389
             cls.lb_member_2_net['id'])
348 390
 
@@ -357,7 +399,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
357 399
         LOG.info('lb_member_2_subnet: {}'.format(cls.lb_member_2_subnet))
358 400
         cls.addClassResourceCleanup(
359 401
             waiters.wait_for_not_found,
360
-            cls.lb_mem_subnet_client.delete_subnet,
402
+            cls._logging_delete_subnet,
361 403
             cls.lb_mem_subnet_client.show_subnet,
362 404
             cls.lb_member_2_subnet['id'])
363 405
 
@@ -378,7 +420,7 @@ class LoadBalancerBaseTest(test.BaseTestCase):
378 420
                 cls.lb_member_2_ipv6_subnet))
379 421
             cls.addClassResourceCleanup(
380 422
                 waiters.wait_for_not_found,
381
-                cls.lb_mem_subnet_client.delete_subnet,
423
+                cls._logging_delete_subnet,
382 424
                 cls.lb_mem_subnet_client.show_subnet,
383 425
                 cls.lb_member_2_ipv6_subnet['id'])
384 426
 

Loading…
Cancel
Save