From 625b5446cba6fb88916b982627cc48924d505301 Mon Sep 17 00:00:00 2001
From: Andrew Hutchings <andrew@linuxjedi.co.uk>
Date: Fri, 5 Jul 2013 20:29:59 +0100
Subject: [PATCH] [API][ADMIN_API] Cycle DB servers if one fails

10 attempts at cycling servers before we give up,
and log when one is failing

Change-Id: Ib8ed63fb3eb1f0b2b4565783950a637a86ed02a4
---
 libra/admin_api/model/lbaas.py | 18 ++++++++++++++++--
 libra/api/model/lbaas.py       | 18 ++++++++++++++++--
 2 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/libra/admin_api/model/lbaas.py b/libra/admin_api/model/lbaas.py
index 52eaa146..58650fb7 100644
--- a/libra/admin_api/model/lbaas.py
+++ b/libra/admin_api/model/lbaas.py
@@ -21,6 +21,7 @@ import random
 import time
 import ConfigParser
 from pecan import conf
+import logging
 
 
 config = ConfigParser.SafeConfigParser()
@@ -163,10 +164,23 @@ class RoutingSession(Session):
 class db_session(object):
     def __init__(self):
         self.session = None
+        self.logger = logging.getLogger(__name__)
 
     def __enter__(self):
-        self.session = sessionmaker(class_=RoutingSession)()
-        return self.session
+        for x in xrange(10):
+            try:
+                self.session = sessionmaker(class_=RoutingSession)()
+                self.session.execute("SELECT 1")
+                return self.session
+            except:
+                self.logger.error(
+                    'Could not connect to DB server: {0}'.format(
+                        RoutingSession.last_engine.url
+                    )
+                )
+                RoutingSession.last_engine = None
+        self.logger.error('Could not connect to any DB server')
+        return None
 
     def __exit__(self, type, value, traceback):
         self.session.close()
diff --git a/libra/api/model/lbaas.py b/libra/api/model/lbaas.py
index 36a33cbc..ae8ed6c4 100644
--- a/libra/api/model/lbaas.py
+++ b/libra/api/model/lbaas.py
@@ -21,6 +21,7 @@ import time
 import random
 import ConfigParser
 from pecan import conf
+import logging
 
 
 config = ConfigParser.SafeConfigParser()
@@ -163,10 +164,23 @@ class RoutingSession(Session):
 class db_session(object):
     def __init__(self):
         self.session = None
+        self.logger = logging.getLogger(__name__)
 
     def __enter__(self):
-        self.session = sessionmaker(class_=RoutingSession)()
-        return self.session
+        for x in xrange(10):
+            try:
+                self.session = sessionmaker(class_=RoutingSession)()
+                self.session.execute("SELECT 1")
+                return self.session
+            except:
+                self.logger.error(
+                    'Could not connect to DB server: {0}'.format(
+                        RoutingSession.last_engine.url
+                    )
+                )
+                RoutingSession.last_engine = None
+        self.logger.error('Could not connect to any DB server')
+        return None
 
     def __exit__(self, type, value, traceback):
         self.session.close()