From 7729f83c2e3ea45883e45f14fabc41c66b10b702 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 10 Jul 2013 09:54:35 +0100 Subject: [PATCH] [API] Retry request on galera deadlocks It is possible in a multi-API situation for galera deadlocks to still happen. This patch will trigger a retry in these scenarios. Change-Id: I3c62a9a39f6dec921dee8106b7a3a98cf1095f6d --- libra/api/controllers/connection_throttle.py | 4 +- libra/api/controllers/health_monitor.py | 4 +- libra/api/controllers/limits.py | 4 +- libra/api/controllers/load_balancers.py | 4 +- libra/api/controllers/logs.py | 4 +- libra/api/controllers/nodes.py | 4 +- libra/api/controllers/session_persistence.py | 4 +- libra/api/controllers/virtualips.py | 4 +- libra/api/library/gearman_client.py | 36 +++++--- libra/api/library/libra_rest_controller.py | 94 ++++++++++++++++++++ 10 files changed, 132 insertions(+), 30 deletions(-) create mode 100644 libra/api/library/libra_rest_controller.py diff --git a/libra/api/controllers/connection_throttle.py b/libra/api/controllers/connection_throttle.py index 5682302e..64d9ef94 100644 --- a/libra/api/controllers/connection_throttle.py +++ b/libra/api/controllers/connection_throttle.py @@ -14,10 +14,10 @@ # under the License. from pecan import response -from pecan.rest import RestController +from libra.api.library.libra_rest_controller import LibraController -class ConnectionThrottleController(RestController): +class ConnectionThrottleController(LibraController): """functions for /loadbalancers/{loadBalancerId}/connectionthrottle/* routing""" diff --git a/libra/api/controllers/health_monitor.py b/libra/api/controllers/health_monitor.py index 740b3c8e..ee81a5c4 100644 --- a/libra/api/controllers/health_monitor.py +++ b/libra/api/controllers/health_monitor.py @@ -14,10 +14,10 @@ # under the License. from pecan import response -from pecan.rest import RestController +from libra.api.library.libra_rest_controller import LibraController -class HealthMonitorController(RestController): +class HealthMonitorController(LibraController): """functions for /loadbalancers/{loadBalancerId}/healthmonitor/* routing""" def get(self, load_balancer_id): diff --git a/libra/api/controllers/limits.py b/libra/api/controllers/limits.py index 677e1926..06002039 100644 --- a/libra/api/controllers/limits.py +++ b/libra/api/controllers/limits.py @@ -14,11 +14,11 @@ # under the License. from pecan import expose -from pecan.rest import RestController +from libra.api.library.libra_rest_controller import LibraController from libra.api.model.lbaas import Limits, db_session -class LimitsController(RestController): +class LimitsController(LibraController): @expose('json') def get(self): resp = {} diff --git a/libra/api/controllers/load_balancers.py b/libra/api/controllers/load_balancers.py index 0b8d1596..6550dc80 100644 --- a/libra/api/controllers/load_balancers.py +++ b/libra/api/controllers/load_balancers.py @@ -15,7 +15,6 @@ import logging # pecan imports from pecan import expose, abort, response, request -from pecan.rest import RestController import wsmeext.pecan as wsme_pecan from wsme.exc import ClientSideError, InvalidInput from wsme import Unset @@ -32,11 +31,12 @@ from libra.api.library.gearman_client import submit_job from libra.api.acl import get_limited_to_project from libra.api.library.exp import OverLimit, IPOutOfRange from libra.api.library.ip_filter import ipfilter +from libra.api.library.libra_rest_controller import LibraController from pecan import conf from sqlalchemy import func -class LoadBalancersController(RestController): +class LoadBalancersController(LibraController): def __init__(self, lbid=None): self.lbid = lbid diff --git a/libra/api/controllers/logs.py b/libra/api/controllers/logs.py index 36e70549..412ec675 100644 --- a/libra/api/controllers/logs.py +++ b/libra/api/controllers/logs.py @@ -14,18 +14,18 @@ # under the License. from pecan import request -from pecan.rest import RestController from pecan import conf import wsmeext.pecan as wsme_pecan from wsme.exc import ClientSideError from wsme import Unset +from libra.api.library.libra_rest_controller import LibraController from libra.api.model.lbaas import LoadBalancer, Device, db_session from libra.api.acl import get_limited_to_project from libra.api.model.validators import LBLogsPost from libra.api.library.gearman_client import submit_job -class LogsController(RestController): +class LogsController(LibraController): def __init__(self, load_balancer_id=None): self.lbid = load_balancer_id diff --git a/libra/api/controllers/nodes.py b/libra/api/controllers/nodes.py index 765ec0e7..b7f5ae28 100644 --- a/libra/api/controllers/nodes.py +++ b/libra/api/controllers/nodes.py @@ -14,11 +14,11 @@ # under the License. from pecan import expose, response, request, abort -from pecan.rest import RestController import wsmeext.pecan as wsme_pecan from wsme.exc import ClientSideError from wsme import Unset #default response objects +from libra.api.library.libra_rest_controller import LibraController from libra.api.model.lbaas import LoadBalancer, Node, db_session, Limits from libra.api.model.lbaas import Device from libra.api.acl import get_limited_to_project @@ -30,7 +30,7 @@ from libra.api.library.ip_filter import ipfilter from pecan import conf -class NodesController(RestController): +class NodesController(LibraController): """Functions for /loadbalancers/{load_balancer_id}/nodes/* routing""" def __init__(self, lbid, nodeid=None): self.lbid = lbid diff --git a/libra/api/controllers/session_persistence.py b/libra/api/controllers/session_persistence.py index 808d639f..d2c0f1fc 100644 --- a/libra/api/controllers/session_persistence.py +++ b/libra/api/controllers/session_persistence.py @@ -14,10 +14,10 @@ # under the License. from pecan import response -from pecan.rest import RestController +from libra.api.library.libra_rest_controller import LibraController -class SessionPersistenceController(RestController): +class SessionPersistenceController(LibraController): """SessionPersistenceController functions for /loadbalancers/{loadBalancerId}/sessionpersistence/* routing """ diff --git a/libra/api/controllers/virtualips.py b/libra/api/controllers/virtualips.py index b5275806..5c5b87f3 100644 --- a/libra/api/controllers/virtualips.py +++ b/libra/api/controllers/virtualips.py @@ -14,12 +14,12 @@ # under the License. from pecan import response, expose, request -from pecan.rest import RestController +from libra.api.library.libra_rest_controller import LibraController from libra.api.model.lbaas import LoadBalancer, Device, db_session from libra.api.acl import get_limited_to_project -class VipsController(RestController): +class VipsController(LibraController): def __init__(self, load_balancer_id=None): self.lbid = load_balancer_id diff --git a/libra/api/library/gearman_client.py b/libra/api/library/gearman_client.py index 52d991e3..dc52fc94 100644 --- a/libra/api/library/gearman_client.py +++ b/libra/api/library/gearman_client.py @@ -18,6 +18,7 @@ import logging from libra.common.json_gearman import JSONGearmanClient from libra.api.model.lbaas import LoadBalancer, db_session, Device from libra.api.model.lbaas import loadbalancers_devices +from sqlalchemy.exc import OperationalError from pecan import conf @@ -38,21 +39,28 @@ def submit_job(job_type, host, data, lbid): def client_job(logger, job_type, host, data, lbid): - try: - client = GearmanClientThread(logger, host, lbid) - logger.info( - "Sending Gearman job {0} to {1} for loadbalancer {2}".format( - job_type, host, lbid + for x in xrange(5): + try: + client = GearmanClientThread(logger, host, lbid) + logger.info( + "Sending Gearman job {0} to {1} for loadbalancer {2}".format( + job_type, host, lbid + ) ) - ) - if job_type == 'UPDATE': - client.send_update(data) - if job_type == 'DELETE': - client.send_delete(data) - if job_type == 'ARCHIVE': - client.send_archive(data) - except: - logger.exception("Gearman thread unhandled exception") + if job_type == 'UPDATE': + client.send_update(data) + if job_type == 'DELETE': + client.send_delete(data) + if job_type == 'ARCHIVE': + client.send_archive(data) + return + except OperationalError as exc: + # Auto retry on galera locking error + logger.warning("Galera deadlock in gearman, retry {0}".format(x+1)) + if exc.args[0] != 1213: + raise + except: + logger.exception("Gearman thread unhandled exception") class GearmanClientThread(object): diff --git a/libra/api/library/libra_rest_controller.py b/libra/api/library/libra_rest_controller.py new file mode 100644 index 00000000..04f6bc26 --- /dev/null +++ b/libra/api/library/libra_rest_controller.py @@ -0,0 +1,94 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +from pecan.rest import RestController +from pecan.core import request, abort +from pecan.decorators import expose +from inspect import getargspec +from webob import exc +from sqlalchemy.exc import OperationalError + + +class LibraController(RestController): + routing_calls = 0 + + @expose() + def _route(self, args): + ''' + Routes a request to the appropriate controller and returns its result. + + Performs a bit of validation - refuses to route delete and put actions + via a GET request). + ''' + # convention uses "_method" to handle browser-unsupported methods + if request.environ.get('pecan.validation_redirected', False) is True: + # + # If the request has been internally redirected due to a validation + # exception, we want the request method to be enforced as GET, not + # the `_method` param which may have been passed for REST support. + # + method = request.method.lower() + else: + method = request.params.get('_method', request.method).lower() + + # make sure DELETE/PUT requests don't use GET + if request.method == 'GET' and method in ('delete', 'put'): + abort(405) + + # check for nested controllers + result = self._find_sub_controllers(args) + if result: + return result + + # handle the request + handler = getattr(self, '_handle_%s' % method, self._handle_custom) + + try: + result = handler(method, args) + + # + # If the signature of the handler does not match the number + # of remaining positional arguments, attempt to handle + # a _lookup method (if it exists) + # + argspec = getargspec(result[0]) + num_args = len(argspec[0][1:]) + if num_args < len(args): + _lookup_result = self._handle_lookup(args) + if _lookup_result: + return _lookup_result + except exc.HTTPNotFound: + # + # If the matching handler results in a 404, attempt to handle + # a _lookup method (if it exists) + # + _lookup_result = self._handle_lookup(args) + if _lookup_result: + return _lookup_result + except OperationalError as sqlexc: + logger = logging.getLogger(__name__) + # if a galera transaction fails due to locking, retry the call + if sqlexc.args[0] == 1213 and LibraController.routing_calls < 5: + LibraController.routing_calls += 1 + logger.warning("Galera deadlock, retry: {0}".format( + LibraController.routing_calls) + ) + result = self._route(args) + else: + raise + + LibraController.routing_calls = 0 + # return the result + return result