Add option sql_max_retries for database connection.

Fixes bug 1019728

also add unit tests for linux bridge plugin config

Change-Id: Ibfea9bac87ba36f301671a35de81577debcd460e

etc/quantum/plugins/linuxbridge/linuxbridge_conf.ini

@@ -9,6 +9,9 @@ vlan_end=3000
 # Replace 127.0.0.1 above with the IP address of the database used by the
 # main quantum server. (Leave it as is if the database runs on this host.)
 sql_connection = sqlite://
+# Database reconnection retry times - in event connectivity is lost
+# set to -1 implies an infinite retry count
+# sql_max_retries = 10
 # Database reconnection interval in seconds - in event connectivity is lost
 reconnect_interval = 2
 

etc/quantum/plugins/openvswitch/ovs_quantum_plugin.ini

@@ -5,6 +5,9 @@
 # Replace 127.0.0.1 above with the IP address of the database used by the
 # main quantum server. (Leave it as is if the database runs on this host.)
 sql_connection = sqlite://
+# Database reconnection retry times - in event connectivity is lost
+# set to -1 implies an infinite retry count
+# sql_max_retries = 10
 # Database reconnection interval in seconds - in event connectivity is lost
 reconnect_interval = 2
 

quantum/db/api.py

@@ -82,7 +82,9 @@ def configure_db(options):
         base = options.get('base', BASE)
         if not register_models(base):
             if 'reconnect_interval' in options:
-                retry_registration(options['reconnect_interval'], base)
+                remaining = options.get('sql_max_retries', -1)
+                reconnect_interval = options['reconnect_interval']
+                retry_registration(remaining, reconnect_interval, base)
 
 
 def clear_db(base=BASE):
@@ -103,10 +105,17 @@ def get_session(autocommit=True, expire_on_commit=False):
     return _MAKER()
 
 
-def retry_registration(reconnect_interval, base=BASE):
+def retry_registration(remaining, reconnect_interval, base=BASE):
+    if remaining == -1:
+        remaining = 'infinite'
     while True:
-        LOG.info("Unable to connect to database. Retrying in %s seconds" %
+        if remaining != 'infinite':
-                 reconnect_interval)
+            if remaining == 0:
+                LOG.error("Database connection lost, exit...")
+                break
+            remaining -= 1
+        LOG.info("Unable to connect to database, %s attempts left. "
+                 "Retrying in %s seconds" % (remaining, reconnect_interval))
         time.sleep(reconnect_interval)
         if register_models(base):
             break

quantum/plugins/linuxbridge/common/config.py

@@ -27,6 +27,7 @@ vlan_opts = [
 
 database_opts = [
     cfg.StrOpt('sql_connection', default='sqlite://'),
+    cfg.IntOpt('sql_max_retries', default=-1),
     cfg.IntOpt('reconnect_interval', default=2),
 ]
 

quantum/plugins/linuxbridge/db/l2network_db.py

@@ -40,6 +40,7 @@ L2_MODEL = l2network_models
 def initialize(base=None):
     global L2_MODEL
     options = {"sql_connection": "%s" % CONF.DATABASE.sql_connection}
+    options.update({"sql_max_retries": CONF.DATABASE.sql_max_retries})
     options.update({"reconnect_interval": CONF.DATABASE.reconnect_interval})
     if base:
         options.update({"base": base})

quantum/plugins/openvswitch/common/config.py

@@ -19,6 +19,7 @@ from quantum.openstack.common import cfg
 
 database_opts = [
     cfg.StrOpt('sql_connection', default='sqlite://'),
+    cfg.IntOpt('sql_max_retries', default=-1),
     cfg.IntOpt('reconnect_interval', default=2),
 ]
 

quantum/plugins/openvswitch/ovs_quantum_plugin.py

@@ -103,6 +103,8 @@ class OVSQuantumPlugin(QuantumPluginBase):
     def __init__(self, configfile=None):
         conf = config.parse(CONF_FILE)
         options = {"sql_connection": conf.DATABASE.sql_connection}
+        sql_max_retries = conf.DATABASE.sql_max_retries
+        options.update({"sql_max_retries": sql_max_retries})
         reconnect_interval = conf.DATABASE.reconnect_interval
         options.update({"reconnect_interval": reconnect_interval})
         db.configure_db(options)
@@ -236,6 +238,8 @@ class OVSQuantumPluginV2(db_base_plugin_v2.QuantumDbPluginV2):
         conf = config.parse(CONF_FILE)
         options = {"sql_connection": conf.DATABASE.sql_connection}
         options.update({'base': models_v2.model_base.BASEV2})
+        sql_max_retries = conf.DATABASE.sql_max_retries
+        options.update({"sql_max_retries": sql_max_retries})
         reconnect_interval = conf.DATABASE.reconnect_interval
         options.update({"reconnect_interval": reconnect_interval})
         db.configure_db(options)

quantum/plugins/ryu/common/config.py

@@ -19,6 +19,7 @@ from quantum.openstack.common import cfg
 
 database_opts = [
     cfg.StrOpt('sql_connection', default='sqlite://'),
+    cfg.IntOpt('sql_max_retries', default=-1),
     cfg.IntOpt('reconnect_interval', default=2),
 ]
 

quantum/plugins/ryu/ovs_quantum_plugin_base.py

@@ -66,6 +66,8 @@ class OVSQuantumPluginBase(QuantumPluginBase):
         LOG.debug("Using configuration file: %s" % configfile)
         conf = config.parse(configfile)
         options = {"sql_connection": conf.DATABASE.sql_connection}
+        sql_max_retries = conf.DATABASE.sql_max_retries
+        options.update({"sql_max_retries": sql_max_retries})
         reconnect_interval = conf.DATABASE.reconnect_interval
         options.update({"reconnect_interval": reconnect_interval})
         db.configure_db(options)

quantum/tests/unit/test_lb_config.py

@@ -25,6 +25,7 @@ class LinuxBridgeConfigTestCase(unittest.TestCase):
     def test_dummy(self):
         configs = """[DATABASE]
 sql_connection = testlink
+sql_max_retries = 200
 reconnect_interval=100
 [AGENT]
 root_helper = mysudo
@@ -39,6 +40,7 @@ polling_interval=50
 
             conf = config.parse(path)
             self.assertEqual('testlink', conf.DATABASE.sql_connection)
+            self.assertEqual(200, conf.DATABASE.sql_max_retries)
             self.assertEqual(100, conf.DATABASE.reconnect_interval)
             self.assertEqual(50, conf.AGENT.polling_interval)
             self.assertEqual('mysudo', conf.AGENT.root_helper)
@@ -59,6 +61,7 @@ polling_interval=50
 
             conf = config.parse(path)
             self.assertEqual('sqlite://', conf.DATABASE.sql_connection)
+            self.assertEqual(-1, conf.DATABASE.sql_max_retries)
             self.assertEqual(2, conf.DATABASE.reconnect_interval)
             self.assertEqual(2, conf.AGENT.polling_interval)
             self.assertEqual('sudo', conf.AGENT.root_helper)

quantum/tests/unit/test_ovs_config.py

@@ -29,6 +29,7 @@ class OvsConfigTestCase(unittest.TestCase):
     def test_tunnel(self):
         configs = """[DATABASE]
 sql_connection = testlink
+sql_max_retries = 200
 reconnect_interval=100
 [OVS]
 enable_tunneling = True
@@ -51,6 +52,7 @@ polling_interval=50
             self.assertEqual('mybrint', conf.OVS.integration_bridge)
             self.assertEqual('mybrtun', conf.OVS.tunnel_bridge)
             self.assertEqual('testlink', conf.DATABASE.sql_connection)
+            self.assertEqual(200, conf.DATABASE.sql_max_retries)
             self.assertEqual(100, conf.DATABASE.reconnect_interval)
             self.assertEqual(50, conf.AGENT.polling_interval)
             self.assertEqual('mysudo', conf.AGENT.root_helper)
@@ -74,6 +76,7 @@ polling_interval=50
             self.assertEqual('br-int', conf.OVS.integration_bridge)
             self.assertEqual('br-tun', conf.OVS.tunnel_bridge)
             self.assertEqual('sqlite://', conf.DATABASE.sql_connection)
+            self.assertEqual(-1, conf.DATABASE.sql_max_retries)
             self.assertEqual(2, conf.DATABASE.reconnect_interval)
             self.assertEqual(2, conf.AGENT.polling_interval)
             self.assertEqual('sudo', conf.AGENT.root_helper)

quantum/tests/unit/test_ryu_config.py

@@ -25,6 +25,7 @@ class RyuConfigTestCase(unittest.TestCase):
     def test_config(self):
         configs = """[DATABASE]
 sql_connection = testlink
+sql_max_retries = 200
 reconnect_interval=100
 [OVS]
 enable_tunneling = True
@@ -44,6 +45,7 @@ polling_interval=50
             conf = config.parse(path)
             self.assertEqual('mybrint', conf.OVS.integration_bridge)
             self.assertEqual('testlink', conf.DATABASE.sql_connection)
+            self.assertEqual(200, conf.DATABASE.sql_max_retries)
             self.assertEqual(100, conf.DATABASE.reconnect_interval)
             self.assertEqual(50, conf.AGENT.polling_interval)
             self.assertEqual('mysudo', conf.AGENT.root_helper)
@@ -65,6 +67,7 @@ polling_interval=50
             conf = config.parse(path)
             self.assertEqual('br-int', conf.OVS.integration_bridge)
             self.assertEqual('sqlite://', conf.DATABASE.sql_connection)
+            self.assertEqual(-1, conf.DATABASE.sql_max_retries)
             self.assertEqual(2, conf.DATABASE.reconnect_interval)
             self.assertEqual(2, conf.AGENT.polling_interval)
             self.assertEqual('sudo', conf.AGENT.root_helper)