Use oslo.db for sqla driver

Use create_engine in oslo.db instead of the sqlalchemy one to get
the optimization benefit from oslo.db.

Breaking change would be replacing database.url with
database.connection therefore database.url will be still
supported until Pike is released. database.url
has been marked as deprecated option.

Change-Id: Id3cdafa791a7d2558a5b065022a9afc6ff31e004
Closes-bug: #1640419

devstack/files/monasca-api/python/api-config.conf

@@ -117,7 +117,7 @@ cluster_ip_addresses: %CASSANDRA_HOST%
 keyspace: monasca
 
 [database]
-url = "%MONASCA_API_DATABASE_URL%"
+connection = "%MONASCA_API_DATABASE_URL%"
 
 [keystone_authtoken]
 identity_uri = http://%KEYSTONE_AUTH_HOST%:%KEYSTONE_AUTH_PORT%

devstack/files/monasca-api/python/api-logging.conf

@@ -40,4 +40,4 @@ formatter = context
 args = ('/var/log/monasca/api/monasca-api.log', 'a', 104857600, 5)
 
 [formatter_context]
-class = oslo_log.formatters.ContextFormatter
+class = oslo_log.formatters.ContextFormatter

etc/api-config.conf

@@ -117,7 +117,8 @@ keyspace: monasca
 
 # Below is configuration for database.
 [database]
-url = "mysql+pymysql://monapi:password@192.168.10.4/mon"
+connection = "mysql+pymysql://monapi:password@192.168.10.4/mon"
+# backend = sqlalchemy
 # host = 192.168.10.4
 # username = monapi
 # password = password

monasca_api/common/repositories/sqla/sql_repository.py

@@ -1,6 +1,5 @@
 # Copyright 2014 Hewlett-Packard
 # Copyright 2016 FUJITSU LIMITED
-# (C) Copyright 2017 Hewlett Packard Enterprise Development LP
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -15,14 +14,55 @@
 # under the License.
 
 from oslo_config import cfg
+from oslo_db.sqlalchemy import enginefacade
 from oslo_log import log
 
-from sqlalchemy.engine.url import URL, make_url
+import sqlalchemy
-from sqlalchemy import MetaData
 
 from monasca_api.common.repositories import exceptions
 
 LOG = log.getLogger(__name__)
+CONF = cfg.CONF
+
+
+def _get_db_conf(conf_group, connection=None):
+    return dict(
+        connection=connection or conf_group.connection,
+        slave_connection=conf_group.slave_connection,
+        sqlite_fk=False,
+        __autocommit=True,
+        expire_on_commit=False,
+        mysql_sql_mode=conf_group.mysql_sql_mode,
+        idle_timeout=conf_group.idle_timeout,
+        connection_debug=conf_group.connection_debug,
+        max_pool_size=conf_group.max_pool_size,
+        max_overflow=conf_group.max_overflow,
+        pool_timeout=conf_group.pool_timeout,
+        sqlite_synchronous=conf_group.sqlite_synchronous,
+        connection_trace=conf_group.connection_trace,
+        max_retries=conf_group.max_retries,
+        retry_interval=conf_group.retry_interval
+    )
+
+
+def create_context_manager(connection=None):
+    """Create a database context manager object.
+
+    :param connection: The database connection string
+    """
+    ctxt_mgr = enginefacade.transaction_context()
+    ctxt_mgr.configure(**_get_db_conf(CONF.database, connection=connection))
+    return ctxt_mgr
+
+
+def get_engine(use_slave=False, connection=None):
+    """Get a database engine object.
+
+    :param use_slave: Whether to use the slave connection
+    :param connection: The database connection string
+    """
+    ctxt_mgr = create_context_manager(connection=connection)
+    return ctxt_mgr.get_legacy_facade().get_engine(use_slave=use_slave)
 
 
 class SQLRepository(object):
@@ -30,23 +70,10 @@ class SQLRepository(object):
     def __init__(self):
 
         try:
-
             super(SQLRepository, self).__init__()
-
+            self.conf = CONF
-            self.conf = cfg.CONF
+            self._db_engine = get_engine()
-            url = None
+            self.metadata = sqlalchemy.MetaData()
-            if self.conf.database.url is not None:
-                url = make_url(self.conf.database.url)
-            else:
-                database_conf = dict(self.conf.database)
-                if 'url' in database_conf:
-                    del database_conf['url']
-                url = URL(**database_conf)
-
-            from sqlalchemy import create_engine
-            self._db_engine = create_engine(url, pool_recycle=3600)
-
-            self.metadata = MetaData()
 
         except Exception as ex:
             LOG.exception(ex)

monasca_api/tests/test_a_repository.py

@@ -20,6 +20,7 @@ import time
 import fixtures
 from oslo_config import cfg
 from oslo_config import fixture as fixture_config
+from oslo_db.sqlalchemy.engines import create_engine
 import testtools
 
 from sqlalchemy import delete, MetaData, insert, bindparam
@@ -31,9 +32,7 @@ CONF = cfg.CONF
 class TestAlarmRepoDB(testtools.TestCase, fixtures.TestWithFixtures):
     @classmethod
     def setUpClass(cls):
-        from sqlalchemy import engine_from_config
+        engine = create_engine('sqlite://')
-
-        engine = engine_from_config({'url': 'sqlite://'}, prefix='')
 
         qry = open('monasca_api/tests/sqlite_alarm.sql', 'r').read()
         sconn = engine.raw_connection()
@@ -174,7 +173,7 @@ class TestAlarmRepoDB(testtools.TestCase, fixtures.TestWithFixtures):
 
         self._fixture_config = self.useFixture(
             fixture_config.Config(cfg.CONF))
-        self._fixture_config.config(url='sqlite://',
+        self._fixture_config.config(connection='sqlite://',
                                     group='database')
 
         from monasca_api.common.repositories.sqla import alarms_repository as ar

monasca_api/tests/test_ad_repository.py

@@ -19,6 +19,7 @@ import datetime
 import fixtures
 from oslo_config import cfg
 from oslo_config import fixture as fixture_config
+from oslo_db.sqlalchemy.engines import create_engine
 from sqlalchemy import delete, MetaData, insert, bindparam, select, func
 import testtools
 
@@ -46,9 +47,7 @@ class TestAlarmDefinitionRepoDB(testtools.TestCase, fixtures.TestWithFixtures):
 
     @classmethod
     def setUpClass(cls):
-        from sqlalchemy import engine_from_config
+        engine = create_engine('sqlite://')
-
-        engine = engine_from_config({'url': 'sqlite://'}, prefix='')
 
         qry = open('monasca_api/tests/sqlite_alarm.sql', 'r').read()
         sconn = engine.raw_connection()
@@ -133,7 +132,7 @@ class TestAlarmDefinitionRepoDB(testtools.TestCase, fixtures.TestWithFixtures):
 
         self._fixture_config = self.useFixture(
             fixture_config.Config(cfg.CONF))
-        self._fixture_config.config(url='sqlite://',
+        self._fixture_config.config(connection='sqlite://',
                                     group='database')
 
         from monasca_api.common.repositories.sqla import alarm_definitions_repository as adr

monasca_api/tests/test_nm_repository.py

@@ -19,6 +19,7 @@ import datetime
 import fixtures
 from oslo_config import cfg
 from oslo_config import fixture as fixture_config
+from oslo_db.sqlalchemy.engines import create_engine
 from sqlalchemy import delete, MetaData, insert, bindparam
 import testtools
 
@@ -30,9 +31,7 @@ CONF = cfg.CONF
 class TestNotificationMethodRepoDB(testtools.TestCase, fixtures.TestWithFixtures):
     @classmethod
     def setUpClass(cls):
-        from sqlalchemy import engine_from_config
+        engine = create_engine('sqlite://')
-
-        engine = engine_from_config({'url': 'sqlite://'}, prefix='')
 
         qry = open('monasca_api/tests/sqlite_alarm.sql', 'r').read()
         sconn = engine.raw_connection()
@@ -71,7 +70,7 @@ class TestNotificationMethodRepoDB(testtools.TestCase, fixtures.TestWithFixtures
 
         self._fixture_config = self.useFixture(
             fixture_config.Config(cfg.CONF))
-        self._fixture_config.config(url='sqlite://',
+        self._fixture_config.config(connection='sqlite://',
                                     group='database')
 
         from monasca_api.common.repositories.sqla import notifications_repository as nr

monasca_api/v2/reference/__init__.py

@@ -16,6 +16,7 @@
 
 from oslo_config import cfg
 from oslo_config import types
+from oslo_db import options
 
 
 """Configurations for reference implementation
@@ -88,6 +89,7 @@ repositories_group = cfg.OptGroup(name='repositories', title='repositories')
 cfg.CONF.register_group(repositories_group)
 cfg.CONF.register_opts(repositories_opts, repositories_group)
 
+
 kafka_opts = [cfg.StrOpt('uri', help='Address to kafka server. For example: '
                                      'uri=192.168.1.191:9092'),
               cfg.StrOpt('metrics_topic', default='metrics',
@@ -140,15 +142,33 @@ cassandra_group = cfg.OptGroup(name='cassandra', title='cassandra')
 cfg.CONF.register_group(cassandra_group)
 cfg.CONF.register_opts(cassandra_opts, cassandra_group)
 
-sql_opts = [cfg.StrOpt('url', default=None),
-            cfg.StrOpt('host', default=None),
-            cfg.StrOpt('username', default=None),
-            cfg.StrOpt('password', default=None, secret=True),
-            cfg.StrOpt('drivername', default=None),
-            cfg.IntOpt('port', default=None),
-            cfg.StrOpt('database', default=None),
-            cfg.StrOpt('query', default=None)]
-sql_group = cfg.OptGroup(name='database', title='sql')
 
-cfg.CONF.register_group(sql_group)
+def register_database_opts():
-cfg.CONF.register_opts(sql_opts, sql_group)
+    # Update the default QueuePool parameters. These can be tweaked by the
+    # conf variables - max_pool_size, max_overflow and pool_timeout
+
+    options.set_defaults(cfg.CONF, connection='sqlite://',
+                         sqlite_db='', max_pool_size=10,
+                         max_overflow=20, pool_timeout=10)
+
+    # register old value
+    url_opt = cfg.StrOpt(name='url',
+                         default=cfg.CONF.database.connection,
+                         required=False,
+                         deprecated_for_removal=True,
+                         deprecated_since='1.6.0',
+                         deprecated_reason=(
+                             'Please use database.connection option,'
+                             'database.url is scheduled for removal '
+                             'in Pike release')
+                         )
+
+    cfg.CONF.register_opts([url_opt], group='database')
+    cfg.CONF.set_override(name='connection', group='database',
+                          override=cfg.CONF.database.url)
+
+register_database_opts()
+
+
+# support URL as an option till Pike is released
+# TODO(trebskit) remove in Pike release

requirements.txt

@@ -1,6 +1,7 @@
 # The order of packages is significant, because pip processes them in the order
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
+oslo.db>=4.15.0  # Apache-2.0
 oslo.config!=3.18.0,>=3.14.0 # Apache-2.0
 oslo.context>=2.12.0 # Apache-2.0
 oslo.log>=3.11.0 # Apache-2.0