Browse Source

Fix deleting multiple resources

Fix an issue that occured when deleting multiple resources at the same
time, the first one was correctly deleted but the other resources failed
because the load balancer was still in a PENDING_UPDATE provisioning
status.

In case of conflicts/PENDING UPDATE provisioning status, added a thread
to defer the deletion of the resource when the LB is active.

Story 2008798
Task 42212

Change-Id: I0ae2472549296c1a299f6f0d73c386dcb65a54f0
changes/80/784880/2
Gregory Thiemonge 4 months ago
parent
commit
4421ff8b6a
  1. 105
      octavia_dashboard/api/rest/lbaasv2.py

105
octavia_dashboard/api/rest/lbaasv2.py

@ -15,7 +15,7 @@
"""
import _thread as thread
from time import sleep
import time
from django.conf import settings
from django.views import generic
@ -27,6 +27,7 @@ try:
from openstack import config as occ
except ImportError:
from os_client_config import config as occ
from openstack import exceptions
from openstack_dashboard.api import neutron
from openstack_dashboard.api.rest import urls
@ -111,7 +112,7 @@ def poll_loadbalancer_status(request, loadbalancer_id, callback,
interval = conf.HORIZON_CONFIG['ajax_poll_interval'] / 1000.0
status = from_state
while status == from_state:
sleep(interval)
time.sleep(interval)
conn = _get_sdk_connection(request)
lb = conn.load_balancer.get_load_balancer(loadbalancer_id)
status = lb.provisioning_status
@ -123,6 +124,68 @@ def poll_loadbalancer_status(request, loadbalancer_id, callback,
callback(request, **kwargs)
def _retry_on_conflict(conn, func, *args, retry_timeout=120, **kwargs):
load_balancer_getter = kwargs.pop('load_balancer_getter')
resource_id = kwargs.pop('resource_id')
interval = conf.HORIZON_CONFIG['ajax_poll_interval'] / 1000.0
load_balancer_id = load_balancer_getter(conn, resource_id)
start = time.time()
while time.time() - start < retry_timeout:
lb = conn.load_balancer.get_load_balancer(load_balancer_id)
if lb.provisioning_status == 'PENDING_UPDATE':
time.sleep(interval)
continue
try:
func(*args, **kwargs)
except exceptions.ConflictException:
# Still catching 409/Conflict as there might have multiple threads
# waiting for a non-PENDING provisioning state
time.sleep(interval)
continue
break
def retry_on_conflict(conn, func, *args, retry_timeout=120, **kwargs):
load_balancer_getter = kwargs.pop('load_balancer_getter')
resource_id = kwargs.pop('resource_id')
try:
func(*args, **kwargs)
except exceptions.ConflictException:
thread.start_new_thread(
_retry_on_conflict,
(conn, func, *args),
{'retry_timeout': retry_timeout,
'load_balancer_getter': load_balancer_getter,
'resource_id': resource_id,
**kwargs})
def listener_get_load_balancer_id(conn, listener_id):
listener = conn.load_balancer.get_listener(listener_id)
return listener.load_balancers[0]['id']
def l7_policy_get_load_balancer_id(conn, l7_policy_id):
l7_policy = conn.load_balancer.get_l7_policy(l7_policy_id)
listener = conn.load_balancer.get_listener(l7_policy.listener_id)
return listener.load_balancers[0]['id']
def pool_get_load_balancer_id(conn, pool_id):
pool = conn.load_balancer.get_pool(pool_id)
return pool.loadbalancers[0]['id']
def health_monitor_get_load_balancer_id(conn, health_monitor_id):
pool_id = conn.load_balancer.get_health_monitor(health_monitor_id)
return pool_get_load_balancer_id(conn, pool_id)
def create_loadbalancer(request):
data = request.DATA
@ -850,7 +913,11 @@ class Listener(generic.View):
http://localhost/api/lbaas/listeners/cc758c90-3d98-4ea1-af44-aab405c9c915
"""
conn = _get_sdk_connection(request)
conn.load_balancer.delete_listener(listener_id, ignore_missing=True)
retry_on_conflict(
conn, conn.load_balancer.delete_listener,
listener_id, ignore_missing=True,
load_balancer_getter=listener_get_load_balancer_id,
resource_id=listener_id)
@urls.register
@ -933,7 +1000,11 @@ class L7Policy(generic.View):
http://localhost/api/lbaas/l7policies/cc758c90-3d98-4ea1-af44-aab405c9c915
"""
conn = _get_sdk_connection(request)
conn.load_balancer.delete_l7_policy(l7_policy_id)
retry_on_conflict(
conn, conn.load_balancer.delete_l7_policy,
l7_policy_id,
load_balancer_getter=l7_policy_get_load_balancer_id,
resource_id=l7_policy_id)
@urls.register
@ -992,7 +1063,11 @@ class L7Rule(generic.View):
def delete(self, request, l7_rule_id, l7_policy_id):
"""Delete a specific l7 rule."""
conn = _get_sdk_connection(request)
conn.load_balancer.delete_l7_rule(l7_rule_id, l7_policy_id)
retry_on_conflict(
conn, conn.load_balancer.delete_l7_rule,
l7_rule_id, l7_policy_id,
load_balancer_getter=l7_policy_get_load_balancer_id,
resource_id=l7_policy_id)
@urls.register
@ -1106,7 +1181,11 @@ class Pool(generic.View):
http://localhost/api/lbaas/pools/cc758c90-3d98-4ea1-af44-aab405c9c915
"""
conn = _get_sdk_connection(request)
conn.load_balancer.delete_pool(pool_id)
retry_on_conflict(
conn, conn.load_balancer.delete_pool,
pool_id,
load_balancer_getter=pool_get_load_balancer_id,
resource_id=pool_id)
@urls.register
@ -1172,7 +1251,11 @@ class Member(generic.View):
"""
conn = _get_sdk_connection(request)
conn.load_balancer.delete_member(member_id, pool_id)
retry_on_conflict(
conn, conn.load_balancer.delete_member,
member_id, pool_id,
load_balancer_getter=pool_get_load_balancer_id,
resource_id=pool_id)
@rest_utils.ajax()
def put(self, request, member_id, pool_id):
@ -1262,8 +1345,12 @@ class HealthMonitor(generic.View):
http://localhost/api/lbaas/healthmonitors/cc758c90-3d98-4ea1-af44-aab405c9c915
"""
conn = _get_sdk_connection(request)
conn.load_balancer.delete_health_monitor(health_monitor_id,
ignore_missing=True)
retry_on_conflict(
conn, conn.load_balancer.delete_health_monitor,
health_monitor_id,
ignore_missing=True,
load_balancer_getter=health_monitor_get_load_balancer_id,
resource_id=health_monitor_id)
@rest_utils.ajax()
def put(self, request, health_monitor_id):

Loading…
Cancel
Save