From bf0b9c90d692afb625e23bc7c71893c7d14bef9f Mon Sep 17 00:00:00 2001
From: Andrew Hutchings <andrew@linuxjedi.co.uk>
Date: Fri, 5 Jul 2013 16:49:01 +0100
Subject: [PATCH] [API][ADMIN_API] Fix small race in SQLAlchemy + galera

It is possible for a session to be cleared before galera replication is
done causing a transaction abort.  Now only switch engines for sessions
after 60 seconds of idle time.  This also gives us a 60 second failover
window.

Change-Id: I8db17a13f460d4be580a170d8cfdcbab40eb6b02
---
 libra/admin_api/model/lbaas.py | 10 ++++++++--
 libra/api/model/lbaas.py       | 10 ++++++++--
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/libra/admin_api/model/lbaas.py b/libra/admin_api/model/lbaas.py
index bc83d6ad..52eaa146 100644
--- a/libra/admin_api/model/lbaas.py
+++ b/libra/admin_api/model/lbaas.py
@@ -18,6 +18,7 @@ from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import relationship, backref, sessionmaker, Session
 import sqlalchemy.types as types
 import random
+import time
 import ConfigParser
 from pecan import conf
 
@@ -140,17 +141,22 @@ class Node(DeclarativeBase):
 
 class RoutingSession(Session):
     """ If an engine is already in use, re-use it.  Otherwise we can end up
-        with deadlocks in Galera, see http://tinyurl.com/9h6qlly """
+        with deadlocks in Galera, see http://tinyurl.com/9h6qlly
+        switch engines every 60 seconds of idle time """
+
     last_engine = None
+    last_engine_time = 0
 
     def get_bind(self, mapper=None, clause=None):
         if (
             RoutingSession.last_engine
-            and RoutingSession.last_engine.pool.checkedout() > 0
+            and time.time() < RoutingSession.last_engine_time + 60
         ):
+            RoutingSession.last_engine_time = time.time()
             return RoutingSession.last_engine
         engine = random.choice(engines)
         RoutingSession.last_engine = engine
+        RoutingSession.last_engine_time = time.time()
         return engine
 
 
diff --git a/libra/api/model/lbaas.py b/libra/api/model/lbaas.py
index d5b1591a..36a33cbc 100644
--- a/libra/api/model/lbaas.py
+++ b/libra/api/model/lbaas.py
@@ -17,6 +17,7 @@ from sqlalchemy import INTEGER, VARCHAR, BIGINT
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import relationship, backref, sessionmaker, Session
 import sqlalchemy.types as types
+import time
 import random
 import ConfigParser
 from pecan import conf
@@ -140,17 +141,22 @@ class Node(DeclarativeBase):
 
 class RoutingSession(Session):
     """ If an engine is already in use, re-use it.  Otherwise we can end up
-        with deadlocks in Galera, see http://tinyurl.com/9h6qlly """
+        with deadlocks in Galera, see http://tinyurl.com/9h6qlly
+        switch engines every 60 seconds of idle time """
+
     last_engine = None
+    last_engine_time = 0
 
     def get_bind(self, mapper=None, clause=None):
         if (
             RoutingSession.last_engine
-            and RoutingSession.last_engine.pool.checkedout() > 0
+            and time.time() < RoutingSession.last_engine_time + 60
         ):
+            RoutingSession.last_engine_time = time.time()
             return RoutingSession.last_engine
         engine = random.choice(engines)
         RoutingSession.last_engine = engine
+        RoutingSession.last_engine_time = time.time()
         return engine