Move sql_xx params to [database] conf section

Update trove configuration files to use the current convention for db
options:
- deprecate the used of [DEFAULT] sql_connection, sql_idle_timeout,
  sql_query_log
- add support for [database] connection, idle_timeout, query_log
- update sample/test conf files to use the new convention

DocImpact: Change to conf options in Install Guide - Install the
  Database service.

This has become more important, as the recent changes to devstack
now use the new [database] section, and the gate is failing due to
that.

Note: On my quick look at the code, there's more work needed to use
oslo_db properly, and it might be better to do that out-of-band
of this fix.

Authored-By: Greg Lucas <glucas@tesora.com>
Co-Authored-By: Peter Stachowski <peter@tesora.com>
Closes-Bug: #1308411
Closes-Bug: #1434856
Change-Id: Ic9e5b20e21d09c4b1d49df660b89decf094440bb

etc/trove/trove-conductor.conf.sample

@@ -2,7 +2,6 @@
 verbose = True
 debug = True
 trove_auth_url = http://0.0.0.0:5000/v2.0
-sql_connection = mysql://root:e1a2c042c828d3566d0a@localhost/trove
 
 #===================== RPC Configuration =================================
 
@@ -52,3 +51,6 @@ conductor_manager = trove.conductor.manager.Manager
 #enabled = False
 # If False doesn't trace SQL requests.
 #trace_sqlalchemy = True
+
+[database]
+connection = mysql://root:e1a2c042c828d3566d0a@localhost/trove

etc/trove/trove-taskmanager.conf.sample

@@ -51,21 +51,6 @@ rabbit_password=f7999d1955c5014aa32c
 # The RabbitMQ virtual host. (string value)
 #rabbit_virtual_host=/
 
-# SQLAlchemy connection string for the reference implementation
-# registry server. Any valid SQLAlchemy connection string is fine.
-# See: http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html#sqlalchemy.create_engine
-sql_connection = mysql://root:e1a2c042c828d3566d0a@localhost/trove
-# sql_connection = mysql://root:root@localhost/trove
-
-# Period in seconds after which SQLAlchemy should reestablish its connection
-# to the database.
-#
-# MySQL uses a default `wait_timeout` of 8 hours, after which it will drop
-# idle connections. This can result in 'MySQL Gone Away' exceptions. If you
-# notice this, you can lower this value to ensure that SQLAlchemy reconnects
-# before MySQL can drop the connection.
-sql_idle_timeout = 3600
-
 #DB Api Implementation
 db_api_implementation = trove.db.sqlalchemy.api
 
@@ -193,6 +178,24 @@ pydev_debug = disabled
 # path to pydevd library. It will be used if pydevd is absent in sys.path
 #pydev_path = <path>
 
+[database]
+
+# SQLAlchemy connection string for the reference implementation
+# registry server. Any valid SQLAlchemy connection string is fine.
+# See: http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html#sqlalchemy.create_engine
+connection = mysql://root:e1a2c042c828d3566d0a@localhost/trove
+# connection = mysql://root:root@localhost/trove
+
+# Period in seconds after which SQLAlchemy should reestablish its connection
+# to the database.
+#
+# MySQL uses a default `wait_timeout` of 8 hours, after which it will drop
+# idle connections. This can result in 'MySQL Gone Away' exceptions. If you
+# notice this, you can lower this value to ensure that SQLAlchemy reconnects
+# before MySQL can drop the connection.
+idle_timeout = 3600
+
+
 # ================= Guestagent related ========================
 #guest_config = /etc/trove/trove-guestagent.conf
 # Use 'guest_info = /etc/guest_info' for pre-Kilo compatibility

etc/trove/trove.conf.sample

@@ -55,22 +55,6 @@ rabbit_password=f7999d1955c5014aa32c
 # The RabbitMQ virtual host. (string value)
 #rabbit_virtual_host=/
 
-# SQLAlchemy connection string for the reference implementation
-# registry server. Any valid SQLAlchemy connection string is fine.
-# See: http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html#sqlalchemy.create_engine
-# sql_connection = sqlite:///trove_test.sqlite
-sql_connection = mysql://root:e1a2c042c828d3566d0a@localhost/trove
-#sql_connection = postgresql://trove:trove@localhost/trove
-
-# Period in seconds after which SQLAlchemy should reestablish its connection
-# to the database.
-#
-# MySQL uses a default `wait_timeout` of 8 hours, after which it will drop
-# idle connections. This can result in 'MySQL Gone Away' exceptions. If you
-# notice this, you can lower this value to ensure that SQLAlchemy reconnects
-# before MySQL can drop the connection.
-sql_idle_timeout = 3600
-
 # Maximum line size of message headers to be accepted.
 # max_header_line may need to be increased when using large tokens
 # (typically those generated by the Keystone v3 API with big service
@@ -188,6 +172,26 @@ api_paste_config = api-paste.ini
 #log_file = trove-api.log
 
 
+[database]
+
+# SQLAlchemy connection string for the reference implementation
+# registry server. Any valid SQLAlchemy connection string is fine.
+# See: http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html#sqlalchemy.create_engine
+# connection = sqlite:///trove_test.sqlite
+connection = mysql://root:e1a2c042c828d3566d0a@localhost/trove
+#connection = postgresql://trove:trove@localhost/trove
+
+# Period in seconds after which SQLAlchemy should reestablish its connection
+# to the database.
+#
+# MySQL uses a default `wait_timeout` of 8 hours, after which it will drop
+# idle connections. This can result in 'MySQL Gone Away' exceptions. If you
+# notice this, you can lower this value to ensure that SQLAlchemy reconnects
+# before MySQL can drop the connection.
+idle_timeout = 3600
+
+
+
 # ============ SSL configuration (and enablement) =============================
 # In order to enable SSL for the trove api server, uncomment
 # the cert_file and key_file - and of course have those files

etc/trove/trove.conf.test

@@ -44,22 +44,6 @@ bind_port = 8779
 # AMQP Connection info
 rabbit_password=f7999d1955c5014aa32c
 
-# SQLAlchemy connection string for the reference implementation
-# registry server. Any valid SQLAlchemy connection string is fine.
-# See: http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html#sqlalchemy.create_engine
-sql_connection = sqlite:///trove_test.sqlite
-#sql_connection = mysql://root:e1a2c042c828d3566d0a@localhost/trove
-#sql_connection = postgresql://trove:trove@localhost/trove
-
-# Period in seconds after which SQLAlchemy should reestablish its connection
-# to the database.
-#
-# MySQL uses a default `wait_timeout` of 8 hours, after which it will drop
-# idle connections. This can result in 'MySQL Gone Away' exceptions. If you
-# notice this, you can lower this value to ensure that SQLAlchemy reconnects
-# before MySQL can drop the connection.
-sql_idle_timeout = 3600
-
 #DB Api Implementation
 db_api_implementation = trove.db.sqlalchemy.api
 
@@ -158,6 +142,23 @@ device_path = /dev/vdb
 volume_support = True
 device_path = /dev/vdb
 
+[database]
+# SQLAlchemy connection string for the reference implementation
+# registry server. Any valid SQLAlchemy connection string is fine.
+# See: http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html#sqlalchemy.create_engine
+connection = sqlite:///trove_test.sqlite
+#connection = mysql://root:e1a2c042c828d3566d0a@localhost/trove
+#connection = postgresql://trove:trove@localhost/trove
+
+# Period in seconds after which SQLAlchemy should reestablish its connection
+# to the database.
+#
+# MySQL uses a default `wait_timeout` of 8 hours, after which it will drop
+# idle connections. This can result in 'MySQL Gone Away' exceptions. If you
+# notice this, you can lower this value to ensure that SQLAlchemy reconnects
+# before MySQL can drop the connection.
+idle_timeout = 3600
+
 [composite:trove]
 use = call:trove.common.wsgi:versioned_urlmap
 /: versions

trove/common/cfg.py

@@ -31,19 +31,6 @@ path_opts = [
 ]
 
 common_opts = [
-    cfg.StrOpt('sql_connection',
-               default='sqlite:///trove_test.sqlite',
-               help='SQL Connection.',
-               secret=True),
-    cfg.IntOpt('sql_idle_timeout', default=3600,
-               help="Idle time (in seconds) after which the connection to the "
-                    "database is reestablished. Some databases will drop "
-                    "connections after a specific amount of idle time. "
-                    "Setting sql_idle_timeout to a lower value than this will "
-                    "ensure that a reconnect occurs before the database can "
-                    "drop the connection."),
-    cfg.BoolOpt('sql_query_log', default=False,
-                help='Write all SQL queries to a log.'),
     cfg.StrOpt('bind_host', default='0.0.0.0',
                help='IP address the API server will listen on.'),
     cfg.IntOpt('bind_port', default=8779,
@@ -406,6 +393,25 @@ profiler_opts = [
                 help="If False doesn't trace SQL requests.")
 ]
 
+
+database_opts = [
+    cfg.StrOpt('connection',
+               default='sqlite:///trove_test.sqlite',
+               help='SQL Connection.',
+               secret=True,
+               deprecated_name='sql_connection',
+               deprecated_group='DEFAULT'),
+    cfg.IntOpt('idle_timeout',
+               default=3600,
+               deprecated_name='sql_idle_timeout',
+               deprecated_group='DEFAULT'),
+    cfg.BoolOpt('query_log',
+                default=False,
+                deprecated_name='sql_query_log',
+                deprecated_group='DEFAULT'),
+]
+
+
 # Datastore specific option groups
 
 # Mysql
@@ -870,6 +876,8 @@ CONF.register_opts(common_opts)
 CONF.register_group(profiler_group)
 CONF.register_opts(profiler_opts, profiler_group)
 
+CONF.register_opts(database_opts, 'database')
+
 CONF.register_group(mysql_group)
 CONF.register_group(percona_group)
 CONF.register_group(redis_group)

trove/db/sqlalchemy/migration.py

@@ -38,7 +38,7 @@ def db_version(options, repo_path=None):
 
     """
     repo_path = get_migrate_repo_path(repo_path)
-    sql_connection = options['sql_connection']
+    sql_connection = options['database']['connection']
     try:
         return versioning_api.db_version(sql_connection, repo_path)
     except versioning_exceptions.DatabaseNotControlledError:
@@ -57,7 +57,7 @@ def upgrade(options, version=None, repo_path=None):
     """
     db_version(options, repo_path)  # Ensure db is under migration control
     repo_path = get_migrate_repo_path(repo_path)
-    sql_connection = options['sql_connection']
+    sql_connection = options['database']['connection']
     version_str = version or 'latest'
     logger.info("Upgrading %(sql_connection)s to version %(version_str)s" %
                 {'sql_connection': sql_connection, 'version_str': version_str})
@@ -74,7 +74,7 @@ def downgrade(options, version, repo_path=None):
     """
     db_version(options, repo_path)  # Ensure db is under migration control
     repo_path = get_migrate_repo_path(repo_path)
-    sql_connection = options['sql_connection']
+    sql_connection = options['database']['connection']
     logger.info("Downgrading %(sql_connection)s to version %(version)s" %
                 {'sql_connection': sql_connection, 'version': version})
     return versioning_api.downgrade(sql_connection, repo_path, version)
@@ -86,7 +86,7 @@ def version_control(options, repo_path=None):
     :param options: options dict
 
     """
-    sql_connection = options['sql_connection']
+    sql_connection = options['database']['connection']
     try:
         _version_control(options)
     except versioning_exceptions.DatabaseAlreadyControlledError:
@@ -102,7 +102,7 @@ def _version_control(options, repo_path):
 
     """
     repo_path = get_migrate_repo_path(repo_path)
-    sql_connection = options['sql_connection']
+    sql_connection = options['database']['connection']
     return versioning_api.version_control(sql_connection, repo_path)
 
 

trove/db/sqlalchemy/session.py

@@ -75,11 +75,11 @@ def configure_db(options, models_mapper=None):
 
 def _create_engine(options):
     engine_args = {
-        "pool_recycle": CONF.sql_idle_timeout,
-        "echo": CONF.sql_query_log
+        "pool_recycle": CONF.database.idle_timeout,
+        "echo": CONF.database.query_log
     }
     LOG.info(_("Creating SQLAlchemy engine with args: %s") % engine_args)
-    db_engine = create_engine(options['sql_connection'], **engine_args)
+    db_engine = create_engine(options['database']['connection'], **engine_args)
     if CONF.profiler.enabled and CONF.profiler.trace_sqlalchemy:
         osprofiler.sqlalchemy.add_tracing(sqlalchemy, db_engine, "db")
     return db_engine