[API] Retry request on galera deadlocks

It is possible in a multi-API situation for galera deadlocks to still happen.
This patch will trigger a retry in these scenarios.

Change-Id: I3c62a9a39f6dec921dee8106b7a3a98cf1095f6d

libra/api/controllers/connection_throttle.py

@@ -14,10 +14,10 @@
 # under the License.
 
 from pecan import response
-from pecan.rest import RestController
+from libra.api.library.libra_rest_controller import LibraController
 
 
-class ConnectionThrottleController(RestController):
+class ConnectionThrottleController(LibraController):
     """functions for /loadbalancers/{loadBalancerId}/connectionthrottle/*
     routing"""
 

libra/api/controllers/health_monitor.py

@@ -14,10 +14,10 @@
 # under the License.
 
 from pecan import response
-from pecan.rest import RestController
+from libra.api.library.libra_rest_controller import LibraController
 
 
-class HealthMonitorController(RestController):
+class HealthMonitorController(LibraController):
     """functions for /loadbalancers/{loadBalancerId}/healthmonitor/* routing"""
 
     def get(self, load_balancer_id):

libra/api/controllers/limits.py

@@ -14,11 +14,11 @@
 # under the License.
 
 from pecan import expose
-from pecan.rest import RestController
+from libra.api.library.libra_rest_controller import LibraController
 from libra.api.model.lbaas import Limits, db_session
 
 
-class LimitsController(RestController):
+class LimitsController(LibraController):
     @expose('json')
     def get(self):
         resp = {}

libra/api/controllers/load_balancers.py

@@ -15,7 +15,6 @@
 import logging
 # pecan imports
 from pecan import expose, abort, response, request
-from pecan.rest import RestController
 import wsmeext.pecan as wsme_pecan
 from wsme.exc import ClientSideError, InvalidInput
 from wsme import Unset
@@ -32,11 +31,12 @@ from libra.api.library.gearman_client import submit_job
 from libra.api.acl import get_limited_to_project
 from libra.api.library.exp import OverLimit, IPOutOfRange
 from libra.api.library.ip_filter import ipfilter
+from libra.api.library.libra_rest_controller import LibraController
 from pecan import conf
 from sqlalchemy import func
 
 
-class LoadBalancersController(RestController):
+class LoadBalancersController(LibraController):
     def __init__(self, lbid=None):
         self.lbid = lbid
 

libra/api/controllers/logs.py

@@ -14,18 +14,18 @@
 # under the License.
 
 from pecan import request
-from pecan.rest import RestController
 from pecan import conf
 import wsmeext.pecan as wsme_pecan
 from wsme.exc import ClientSideError
 from wsme import Unset
+from libra.api.library.libra_rest_controller import LibraController
 from libra.api.model.lbaas import LoadBalancer, Device, db_session
 from libra.api.acl import get_limited_to_project
 from libra.api.model.validators import LBLogsPost
 from libra.api.library.gearman_client import submit_job
 
 
-class LogsController(RestController):
+class LogsController(LibraController):
     def __init__(self, load_balancer_id=None):
         self.lbid = load_balancer_id
 

libra/api/controllers/nodes.py

@@ -14,11 +14,11 @@
 # under the License.
 
 from pecan import expose, response, request, abort
-from pecan.rest import RestController
 import wsmeext.pecan as wsme_pecan
 from wsme.exc import ClientSideError
 from wsme import Unset
 #default response objects
+from libra.api.library.libra_rest_controller import LibraController
 from libra.api.model.lbaas import LoadBalancer, Node, db_session, Limits
 from libra.api.model.lbaas import Device
 from libra.api.acl import get_limited_to_project
@@ -30,7 +30,7 @@ from libra.api.library.ip_filter import ipfilter
 from pecan import conf
 
 
-class NodesController(RestController):
+class NodesController(LibraController):
     """Functions for /loadbalancers/{load_balancer_id}/nodes/* routing"""
     def __init__(self, lbid, nodeid=None):
         self.lbid = lbid

libra/api/controllers/session_persistence.py

@@ -14,10 +14,10 @@
 # under the License.
 
 from pecan import response
-from pecan.rest import RestController
+from libra.api.library.libra_rest_controller import LibraController
 
 
-class SessionPersistenceController(RestController):
+class SessionPersistenceController(LibraController):
     """SessionPersistenceController
     functions for /loadbalancers/{loadBalancerId}/sessionpersistence/* routing
     """

libra/api/controllers/virtualips.py

@@ -14,12 +14,12 @@
 # under the License.
 
 from pecan import response, expose, request
-from pecan.rest import RestController
+from libra.api.library.libra_rest_controller import LibraController
 from libra.api.model.lbaas import LoadBalancer, Device, db_session
 from libra.api.acl import get_limited_to_project
 
 
-class VipsController(RestController):
+class VipsController(LibraController):
     def __init__(self, load_balancer_id=None):
         self.lbid = load_balancer_id
 

libra/api/library/gearman_client.py

@@ -18,6 +18,7 @@ import logging
 from libra.common.json_gearman import JSONGearmanClient
 from libra.api.model.lbaas import LoadBalancer, db_session, Device
 from libra.api.model.lbaas import loadbalancers_devices
+from sqlalchemy.exc import OperationalError
 from pecan import conf
 
 
@@ -38,21 +39,28 @@ def submit_job(job_type, host, data, lbid):
 
 
 def client_job(logger, job_type, host, data, lbid):
-    try:
+    for x in xrange(5):
-        client = GearmanClientThread(logger, host, lbid)
+        try:
-        logger.info(
+            client = GearmanClientThread(logger, host, lbid)
-            "Sending Gearman job {0} to {1} for loadbalancer {2}".format(
+            logger.info(
-                job_type, host, lbid
+                "Sending Gearman job {0} to {1} for loadbalancer {2}".format(
+                    job_type, host, lbid
+                )
             )
-        )
+            if job_type == 'UPDATE':
-        if job_type == 'UPDATE':
+                client.send_update(data)
-            client.send_update(data)
+            if job_type == 'DELETE':
-        if job_type == 'DELETE':
+                client.send_delete(data)
-            client.send_delete(data)
+            if job_type == 'ARCHIVE':
-        if job_type == 'ARCHIVE':
+                client.send_archive(data)
-            client.send_archive(data)
+            return
-    except:
+        except OperationalError as exc:
-        logger.exception("Gearman thread unhandled exception")
+            # Auto retry on galera locking error
+            logger.warning("Galera deadlock in gearman, retry {0}".format(x+1))
+            if exc.args[0] != 1213:
+                raise
+        except:
+            logger.exception("Gearman thread unhandled exception")
 
 
 class GearmanClientThread(object):

libra/api/library/libra_rest_controller.py

@@ -0,0 +1,94 @@
+# Copyright 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+from pecan.rest import RestController
+from pecan.core import request, abort
+from pecan.decorators import expose
+from inspect import getargspec
+from webob import exc
+from sqlalchemy.exc import OperationalError
+
+
+class LibraController(RestController):
+    routing_calls = 0
+
+    @expose()
+    def _route(self, args):
+        '''
+        Routes a request to the appropriate controller and returns its result.
+
+        Performs a bit of validation - refuses to route delete and put actions
+        via a GET request).
+        '''
+        # convention uses "_method" to handle browser-unsupported methods
+        if request.environ.get('pecan.validation_redirected', False) is True:
+            #
+            # If the request has been internally redirected due to a validation
+            # exception, we want the request method to be enforced as GET, not
+            # the `_method` param which may have been passed for REST support.
+            #
+            method = request.method.lower()
+        else:
+            method = request.params.get('_method', request.method).lower()
+
+        # make sure DELETE/PUT requests don't use GET
+        if request.method == 'GET' and method in ('delete', 'put'):
+            abort(405)
+
+        # check for nested controllers
+        result = self._find_sub_controllers(args)
+        if result:
+            return result
+
+        # handle the request
+        handler = getattr(self, '_handle_%s' % method, self._handle_custom)
+
+        try:
+            result = handler(method, args)
+
+            #
+            # If the signature of the handler does not match the number
+            # of remaining positional arguments, attempt to handle
+            # a _lookup method (if it exists)
+            #
+            argspec = getargspec(result[0])
+            num_args = len(argspec[0][1:])
+            if num_args < len(args):
+                _lookup_result = self._handle_lookup(args)
+                if _lookup_result:
+                    return _lookup_result
+        except exc.HTTPNotFound:
+            #
+            # If the matching handler results in a 404, attempt to handle
+            # a _lookup method (if it exists)
+            #
+            _lookup_result = self._handle_lookup(args)
+            if _lookup_result:
+                return _lookup_result
+        except OperationalError as sqlexc:
+            logger = logging.getLogger(__name__)
+            # if a galera transaction fails due to locking, retry the call
+            if sqlexc.args[0] == 1213 and LibraController.routing_calls < 5:
+                LibraController.routing_calls += 1
+                logger.warning("Galera deadlock, retry: {0}".format(
+                    LibraController.routing_calls)
+                )
+                result = self._route(args)
+            else:
+                raise
+
+        LibraController.routing_calls = 0
+        # return the result
+        return result