diff --git a/octavia_dashboard/api/rest/lbaasv2.py b/octavia_dashboard/api/rest/lbaasv2.py index 2b70563c..d089213d 100644 --- a/octavia_dashboard/api/rest/lbaasv2.py +++ b/octavia_dashboard/api/rest/lbaasv2.py @@ -15,7 +15,7 @@ """ import _thread as thread -from time import sleep +import time from django.conf import settings from django.views import generic @@ -27,6 +27,7 @@ try: from openstack import config as occ except ImportError: from os_client_config import config as occ +from openstack import exceptions from openstack_dashboard.api import neutron from openstack_dashboard.api.rest import urls @@ -111,7 +112,7 @@ def poll_loadbalancer_status(request, loadbalancer_id, callback, interval = conf.HORIZON_CONFIG['ajax_poll_interval'] / 1000.0 status = from_state while status == from_state: - sleep(interval) + time.sleep(interval) conn = _get_sdk_connection(request) lb = conn.load_balancer.get_load_balancer(loadbalancer_id) status = lb.provisioning_status @@ -123,6 +124,67 @@ def poll_loadbalancer_status(request, loadbalancer_id, callback, callback(request, **kwargs) +def _retry_on_conflict(conn, func, *args, retry_timeout=120, **kwargs): + load_balancer_getter = kwargs.pop('load_balancer_getter') + resource_id = kwargs.pop('resource_id') + + interval = conf.HORIZON_CONFIG['ajax_poll_interval'] / 1000.0 + + load_balancer_id = load_balancer_getter(conn, resource_id) + + start = time.time() + while time.time() - start < retry_timeout: + lb = conn.load_balancer.get_load_balancer(load_balancer_id) + if lb.provisioning_status == 'PENDING_UPDATE': + time.sleep(interval) + continue + + try: + func(*args, **kwargs) + except exceptions.ConflictException: + # Still catching 409/Conflict as there might have multiple threads + # waiting for a non-PENDING provisioning state + continue + break + + +def retry_on_conflict(conn, func, *args, retry_timeout=120, **kwargs): + load_balancer_getter = kwargs.pop('load_balancer_getter') + resource_id = kwargs.pop('resource_id') + + try: + func(*args, **kwargs) + except exceptions.ConflictException: + thread.start_new_thread( + _retry_on_conflict, + (conn, func, *args), + {'retry_timeout': retry_timeout, + 'load_balancer_getter': load_balancer_getter, + 'resource_id': resource_id, + **kwargs}) + + +def listener_get_load_balancer_id(conn, listener_id): + listener = conn.load_balancer.get_listener(listener_id) + return listener.load_balancers[0]['id'] + + +def l7_policy_get_load_balancer_id(conn, l7_policy_id): + l7_policy = conn.load_balancer.get_l7_policy(l7_policy_id) + listener = conn.load_balancer.get_listener(l7_policy.listener_id) + return listener.load_balancers[0]['id'] + + +def pool_get_load_balancer_id(conn, pool_id): + pool = conn.load_balancer.get_pool(pool_id) + return pool.loadbalancers[0]['id'] + + +def health_monitor_get_load_balancer_id(conn, health_monitor_id): + pool_id = conn.load_balancer.get_health_monitor(health_monitor_id) + return pool_get_load_balancer_id(conn, pool_id) + + def create_loadbalancer(request): data = request.DATA @@ -850,7 +912,11 @@ class Listener(generic.View): http://localhost/api/lbaas/listeners/cc758c90-3d98-4ea1-af44-aab405c9c915 """ conn = _get_sdk_connection(request) - conn.load_balancer.delete_listener(listener_id, ignore_missing=True) + retry_on_conflict( + conn, conn.load_balancer.delete_listener, + listener_id, ignore_missing=True, + load_balancer_getter=listener_get_load_balancer_id, + resource_id=listener_id) @urls.register @@ -933,7 +999,11 @@ class L7Policy(generic.View): http://localhost/api/lbaas/l7policies/cc758c90-3d98-4ea1-af44-aab405c9c915 """ conn = _get_sdk_connection(request) - conn.load_balancer.delete_l7_policy(l7_policy_id) + retry_on_conflict( + conn, conn.load_balancer.delete_l7_policy, + l7_policy_id, + load_balancer_getter=l7_policy_get_load_balancer_id, + resource_id=l7_policy_id) @urls.register @@ -992,7 +1062,11 @@ class L7Rule(generic.View): def delete(self, request, l7_rule_id, l7_policy_id): """Delete a specific l7 rule.""" conn = _get_sdk_connection(request) - conn.load_balancer.delete_l7_rule(l7_rule_id, l7_policy_id) + retry_on_conflict( + conn, conn.load_balancer.delete_l7_rule, + l7_rule_id, l7_policy_id, + load_balancer_getter=l7_policy_get_load_balancer_id, + resource_id=l7_policy_id) @urls.register @@ -1106,7 +1180,11 @@ class Pool(generic.View): http://localhost/api/lbaas/pools/cc758c90-3d98-4ea1-af44-aab405c9c915 """ conn = _get_sdk_connection(request) - conn.load_balancer.delete_pool(pool_id) + retry_on_conflict( + conn, conn.load_balancer.delete_pool, + pool_id, + load_balancer_getter=pool_get_load_balancer_id, + resource_id=pool_id) @urls.register @@ -1172,7 +1250,11 @@ class Member(generic.View): """ conn = _get_sdk_connection(request) - conn.load_balancer.delete_member(member_id, pool_id) + retry_on_conflict( + conn, conn.load_balancer.delete_member, + member_id, pool_id, + load_balancer_getter=pool_get_load_balancer_id, + resource_id=pool_id) @rest_utils.ajax() def put(self, request, member_id, pool_id): @@ -1262,8 +1344,12 @@ class HealthMonitor(generic.View): http://localhost/api/lbaas/healthmonitors/cc758c90-3d98-4ea1-af44-aab405c9c915 """ conn = _get_sdk_connection(request) - conn.load_balancer.delete_health_monitor(health_monitor_id, - ignore_missing=True) + retry_on_conflict( + conn, conn.load_balancer.delete_health_monitor, + health_monitor_id, + ignore_missing=True, + load_balancer_getter=health_monitor_get_load_balancer_id, + resource_id=health_monitor_id) @rest_utils.ajax() def put(self, request, health_monitor_id):