From f379d71fcb165b9ebe7ea751b57b28959c38c2e8 Mon Sep 17 00:00:00 2001 From: liyingjun Date: Thu, 10 Nov 2016 10:10:07 +0800 Subject: [PATCH] Use oslo.db for sqla driver Use create_engine in oslo.db instead of the sqlalchemy one to get the optimization benefit from oslo.db. Breaking change would be replacing database.url with database.connection therefore database.url will be still supported until Pike is released. database.url has been marked as deprecated option. Change-Id: Id3cdafa791a7d2558a5b065022a9afc6ff31e004 Closes-bug: #1640419 --- .../files/monasca-api/python/api-config.conf | 2 +- .../files/monasca-api/python/api-logging.conf | 2 +- etc/api-config.conf | 3 +- .../repositories/sqla/sql_repository.py | 65 +++++++++++++------ monasca_api/tests/test_a_repository.py | 7 +- monasca_api/tests/test_ad_repository.py | 7 +- monasca_api/tests/test_nm_repository.py | 7 +- monasca_api/v2/reference/__init__.py | 42 ++++++++---- requirements.txt | 1 + 9 files changed, 91 insertions(+), 45 deletions(-) diff --git a/devstack/files/monasca-api/python/api-config.conf b/devstack/files/monasca-api/python/api-config.conf index cbf6f5564..72206733d 100644 --- a/devstack/files/monasca-api/python/api-config.conf +++ b/devstack/files/monasca-api/python/api-config.conf @@ -117,7 +117,7 @@ cluster_ip_addresses: %CASSANDRA_HOST% keyspace: monasca [database] -url = "%MONASCA_API_DATABASE_URL%" +connection = "%MONASCA_API_DATABASE_URL%" [keystone_authtoken] identity_uri = http://%KEYSTONE_AUTH_HOST%:%KEYSTONE_AUTH_PORT% diff --git a/devstack/files/monasca-api/python/api-logging.conf b/devstack/files/monasca-api/python/api-logging.conf index 0c54137d3..ca864f289 100644 --- a/devstack/files/monasca-api/python/api-logging.conf +++ b/devstack/files/monasca-api/python/api-logging.conf @@ -40,4 +40,4 @@ formatter = context args = ('/var/log/monasca/api/monasca-api.log', 'a', 104857600, 5) [formatter_context] -class = oslo_log.formatters.ContextFormatter \ No newline at end of file +class = oslo_log.formatters.ContextFormatter diff --git a/etc/api-config.conf b/etc/api-config.conf index b56386fb2..42de7f722 100755 --- a/etc/api-config.conf +++ b/etc/api-config.conf @@ -117,7 +117,8 @@ keyspace: monasca # Below is configuration for database. [database] -url = "mysql+pymysql://monapi:password@192.168.10.4/mon" +connection = "mysql+pymysql://monapi:password@192.168.10.4/mon" +# backend = sqlalchemy # host = 192.168.10.4 # username = monapi # password = password diff --git a/monasca_api/common/repositories/sqla/sql_repository.py b/monasca_api/common/repositories/sqla/sql_repository.py index a2d0fae98..e24b8914c 100644 --- a/monasca_api/common/repositories/sqla/sql_repository.py +++ b/monasca_api/common/repositories/sqla/sql_repository.py @@ -1,6 +1,5 @@ # Copyright 2014 Hewlett-Packard # Copyright 2016 FUJITSU LIMITED -# (C) Copyright 2017 Hewlett Packard Enterprise Development LP # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -15,14 +14,55 @@ # under the License. from oslo_config import cfg +from oslo_db.sqlalchemy import enginefacade from oslo_log import log -from sqlalchemy.engine.url import URL, make_url -from sqlalchemy import MetaData +import sqlalchemy from monasca_api.common.repositories import exceptions LOG = log.getLogger(__name__) +CONF = cfg.CONF + + +def _get_db_conf(conf_group, connection=None): + return dict( + connection=connection or conf_group.connection, + slave_connection=conf_group.slave_connection, + sqlite_fk=False, + __autocommit=True, + expire_on_commit=False, + mysql_sql_mode=conf_group.mysql_sql_mode, + idle_timeout=conf_group.idle_timeout, + connection_debug=conf_group.connection_debug, + max_pool_size=conf_group.max_pool_size, + max_overflow=conf_group.max_overflow, + pool_timeout=conf_group.pool_timeout, + sqlite_synchronous=conf_group.sqlite_synchronous, + connection_trace=conf_group.connection_trace, + max_retries=conf_group.max_retries, + retry_interval=conf_group.retry_interval + ) + + +def create_context_manager(connection=None): + """Create a database context manager object. + + :param connection: The database connection string + """ + ctxt_mgr = enginefacade.transaction_context() + ctxt_mgr.configure(**_get_db_conf(CONF.database, connection=connection)) + return ctxt_mgr + + +def get_engine(use_slave=False, connection=None): + """Get a database engine object. + + :param use_slave: Whether to use the slave connection + :param connection: The database connection string + """ + ctxt_mgr = create_context_manager(connection=connection) + return ctxt_mgr.get_legacy_facade().get_engine(use_slave=use_slave) class SQLRepository(object): @@ -30,23 +70,10 @@ class SQLRepository(object): def __init__(self): try: - super(SQLRepository, self).__init__() - - self.conf = cfg.CONF - url = None - if self.conf.database.url is not None: - url = make_url(self.conf.database.url) - else: - database_conf = dict(self.conf.database) - if 'url' in database_conf: - del database_conf['url'] - url = URL(**database_conf) - - from sqlalchemy import create_engine - self._db_engine = create_engine(url, pool_recycle=3600) - - self.metadata = MetaData() + self.conf = CONF + self._db_engine = get_engine() + self.metadata = sqlalchemy.MetaData() except Exception as ex: LOG.exception(ex) diff --git a/monasca_api/tests/test_a_repository.py b/monasca_api/tests/test_a_repository.py index 0381f9114..ee104bab0 100644 --- a/monasca_api/tests/test_a_repository.py +++ b/monasca_api/tests/test_a_repository.py @@ -20,6 +20,7 @@ import time import fixtures from oslo_config import cfg from oslo_config import fixture as fixture_config +from oslo_db.sqlalchemy.engines import create_engine import testtools from sqlalchemy import delete, MetaData, insert, bindparam @@ -31,9 +32,7 @@ CONF = cfg.CONF class TestAlarmRepoDB(testtools.TestCase, fixtures.TestWithFixtures): @classmethod def setUpClass(cls): - from sqlalchemy import engine_from_config - - engine = engine_from_config({'url': 'sqlite://'}, prefix='') + engine = create_engine('sqlite://') qry = open('monasca_api/tests/sqlite_alarm.sql', 'r').read() sconn = engine.raw_connection() @@ -174,7 +173,7 @@ class TestAlarmRepoDB(testtools.TestCase, fixtures.TestWithFixtures): self._fixture_config = self.useFixture( fixture_config.Config(cfg.CONF)) - self._fixture_config.config(url='sqlite://', + self._fixture_config.config(connection='sqlite://', group='database') from monasca_api.common.repositories.sqla import alarms_repository as ar diff --git a/monasca_api/tests/test_ad_repository.py b/monasca_api/tests/test_ad_repository.py index 710666baa..6db9bfff9 100644 --- a/monasca_api/tests/test_ad_repository.py +++ b/monasca_api/tests/test_ad_repository.py @@ -19,6 +19,7 @@ import datetime import fixtures from oslo_config import cfg from oslo_config import fixture as fixture_config +from oslo_db.sqlalchemy.engines import create_engine from sqlalchemy import delete, MetaData, insert, bindparam, select, func import testtools @@ -46,9 +47,7 @@ class TestAlarmDefinitionRepoDB(testtools.TestCase, fixtures.TestWithFixtures): @classmethod def setUpClass(cls): - from sqlalchemy import engine_from_config - - engine = engine_from_config({'url': 'sqlite://'}, prefix='') + engine = create_engine('sqlite://') qry = open('monasca_api/tests/sqlite_alarm.sql', 'r').read() sconn = engine.raw_connection() @@ -133,7 +132,7 @@ class TestAlarmDefinitionRepoDB(testtools.TestCase, fixtures.TestWithFixtures): self._fixture_config = self.useFixture( fixture_config.Config(cfg.CONF)) - self._fixture_config.config(url='sqlite://', + self._fixture_config.config(connection='sqlite://', group='database') from monasca_api.common.repositories.sqla import alarm_definitions_repository as adr diff --git a/monasca_api/tests/test_nm_repository.py b/monasca_api/tests/test_nm_repository.py index b374d7344..a9285d727 100644 --- a/monasca_api/tests/test_nm_repository.py +++ b/monasca_api/tests/test_nm_repository.py @@ -19,6 +19,7 @@ import datetime import fixtures from oslo_config import cfg from oslo_config import fixture as fixture_config +from oslo_db.sqlalchemy.engines import create_engine from sqlalchemy import delete, MetaData, insert, bindparam import testtools @@ -30,9 +31,7 @@ CONF = cfg.CONF class TestNotificationMethodRepoDB(testtools.TestCase, fixtures.TestWithFixtures): @classmethod def setUpClass(cls): - from sqlalchemy import engine_from_config - - engine = engine_from_config({'url': 'sqlite://'}, prefix='') + engine = create_engine('sqlite://') qry = open('monasca_api/tests/sqlite_alarm.sql', 'r').read() sconn = engine.raw_connection() @@ -71,7 +70,7 @@ class TestNotificationMethodRepoDB(testtools.TestCase, fixtures.TestWithFixtures self._fixture_config = self.useFixture( fixture_config.Config(cfg.CONF)) - self._fixture_config.config(url='sqlite://', + self._fixture_config.config(connection='sqlite://', group='database') from monasca_api.common.repositories.sqla import notifications_repository as nr diff --git a/monasca_api/v2/reference/__init__.py b/monasca_api/v2/reference/__init__.py index 76bc2f0ad..13ceaeb78 100644 --- a/monasca_api/v2/reference/__init__.py +++ b/monasca_api/v2/reference/__init__.py @@ -16,6 +16,7 @@ from oslo_config import cfg from oslo_config import types +from oslo_db import options """Configurations for reference implementation @@ -88,6 +89,7 @@ repositories_group = cfg.OptGroup(name='repositories', title='repositories') cfg.CONF.register_group(repositories_group) cfg.CONF.register_opts(repositories_opts, repositories_group) + kafka_opts = [cfg.StrOpt('uri', help='Address to kafka server. For example: ' 'uri=192.168.1.191:9092'), cfg.StrOpt('metrics_topic', default='metrics', @@ -140,15 +142,33 @@ cassandra_group = cfg.OptGroup(name='cassandra', title='cassandra') cfg.CONF.register_group(cassandra_group) cfg.CONF.register_opts(cassandra_opts, cassandra_group) -sql_opts = [cfg.StrOpt('url', default=None), - cfg.StrOpt('host', default=None), - cfg.StrOpt('username', default=None), - cfg.StrOpt('password', default=None, secret=True), - cfg.StrOpt('drivername', default=None), - cfg.IntOpt('port', default=None), - cfg.StrOpt('database', default=None), - cfg.StrOpt('query', default=None)] -sql_group = cfg.OptGroup(name='database', title='sql') -cfg.CONF.register_group(sql_group) -cfg.CONF.register_opts(sql_opts, sql_group) +def register_database_opts(): + # Update the default QueuePool parameters. These can be tweaked by the + # conf variables - max_pool_size, max_overflow and pool_timeout + + options.set_defaults(cfg.CONF, connection='sqlite://', + sqlite_db='', max_pool_size=10, + max_overflow=20, pool_timeout=10) + + # register old value + url_opt = cfg.StrOpt(name='url', + default=cfg.CONF.database.connection, + required=False, + deprecated_for_removal=True, + deprecated_since='1.6.0', + deprecated_reason=( + 'Please use database.connection option,' + 'database.url is scheduled for removal ' + 'in Pike release') + ) + + cfg.CONF.register_opts([url_opt], group='database') + cfg.CONF.set_override(name='connection', group='database', + override=cfg.CONF.database.url) + +register_database_opts() + + +# support URL as an option till Pike is released +# TODO(trebskit) remove in Pike release diff --git a/requirements.txt b/requirements.txt index a62a58772..c6b40b426 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. +oslo.db>=4.15.0 # Apache-2.0 oslo.config!=3.18.0,>=3.14.0 # Apache-2.0 oslo.context>=2.12.0 # Apache-2.0 oslo.log>=3.11.0 # Apache-2.0