From 5a57f15dedacdf73eef619d7093841593862ec89 Mon Sep 17 00:00:00 2001 From: Serg Melikyan Date: Mon, 17 Jun 2013 15:37:44 +0400 Subject: [PATCH] Move to openstack.common.db Change-Id: I74a2e1d25b378b57e2e0ba181eee95bf96a5afea --- etc/murano-api.conf | 7 +- muranoapi/common/config.py | 13 +- muranoapi/db/__init__.py | 13 -- muranoapi/db/session.py | 158 ++---------------------- muranoapi/openstack/common/log.py | 2 +- muranoapi/openstack/common/timeutils.py | 21 ++-- 6 files changed, 29 insertions(+), 185 deletions(-) diff --git a/etc/murano-api.conf b/etc/murano-api.conf index c573dc10..8175accc 100644 --- a/etc/murano-api.conf +++ b/etc/murano-api.conf @@ -14,8 +14,13 @@ bind_port = 8082 # Log to this file. Make sure the user has permissions to write to this file! log_file = /tmp/murano-api.log +[database] #A valid SQLAlchemy connection string for the metadata database -sql_connection = sqlite:///murano.sqlite +#connection = mysql://root:password@localhost:3306/murano +connection = sqlite:///murano.sqlite + +#A boolean that determines if the database will be automatically created +auto_create = True #A boolean that determines if the database will be automatically created db_auto_create = True diff --git a/muranoapi/common/config.py b/muranoapi/common/config.py index cc2e1cbd..196d79a6 100644 --- a/muranoapi/common/config.py +++ b/muranoapi/common/config.py @@ -59,16 +59,7 @@ rabbit_opts = [ ] db_opts = [ - cfg.IntOpt('sql_idle_timeout', default=3600, - help=_('Period in seconds after which SQLAlchemy should ' - 'reestablish its connection to the database.')), - cfg.IntOpt('sql_max_retries', default=60, - help=_('The number of times to retry a connection to the SQL' - 'server.')), - cfg.IntOpt('sql_retry_interval', default=1, - help=_('The amount of time to wait (in seconds) before ' - 'attempting to retry the SQL connection.')), - cfg.BoolOpt('db_auto_create', default=False, + cfg.BoolOpt('auto_create', default=False, help=_('A boolean that determines if the database will be ' 'automatically created.')), ] @@ -78,7 +69,7 @@ CONF.register_opts(paste_deploy_opts, group='paste_deploy') CONF.register_cli_opts(bind_opts) CONF.register_opts(reports_opts, group='reports') CONF.register_opts(rabbit_opts, group='rabbitmq') -CONF.register_opts(db_opts) +CONF.register_opts(db_opts, group='database') CONF.import_opt('verbose', 'muranoapi.openstack.common.log') diff --git a/muranoapi/db/__init__.py b/muranoapi/db/__init__.py index 9ee4d343..7d93825c 100644 --- a/muranoapi/db/__init__.py +++ b/muranoapi/db/__init__.py @@ -11,16 +11,3 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - -from oslo.config import cfg - -sql_connection_opt = cfg.StrOpt('sql_connection', - default='sqlite:///muranoapi.sqlite', - secret=True, - metavar='CONNECTION', - help='A valid SQLAlchemy connection ' - 'string for the metadata database. ' - 'Default: %(default)s') - -CONF = cfg.CONF -CONF.register_opt(sql_connection_opt) diff --git a/muranoapi/db/session.py b/muranoapi/db/session.py index 3ee21e03..046246fa 100644 --- a/muranoapi/db/session.py +++ b/muranoapi/db/session.py @@ -20,172 +20,32 @@ """Session management functions.""" import os -import time -import sqlalchemy -import logging from migrate.versioning import api as versioning_api from migrate import exceptions as versioning_exceptions -from sqlalchemy.exc import DisconnectionError -from sqlalchemy.orm import sessionmaker +from muranoapi.openstack.common.db.sqlalchemy import session from muranoapi.common.config import CONF as conf +from muranoapi.openstack.common import log as logging from muranoapi.db import migrate_repo -from muranoapi.openstack.common import log as mlogging - -_ENGINE = None -_MAKER = None -_MAX_RETRIES = None -_RETRY_INTERVAL = None -_IDLE_TIMEOUT = None -_CONNECTION = None - -sa_logger = None -log = mlogging.getLogger(__name__) - - -def _ping_listener(dbapi_conn, connection_rec, connection_proxy): - - """ - Ensures that MySQL connections checked out of the - pool are alive. - - Borrowed from: - http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f - """ - - try: - dbapi_conn.cursor().execute('select 1') - except dbapi_conn.OperationalError as ex: - if ex.args[0] in (2006, 2013, 2014, 2045, 2055): - msg = 'Got mysql server has gone away: %s' % ex - log.warn(msg) - raise DisconnectionError(msg) - else: - raise - - -def setup_db_env(): - """ - Setup configuration for database - """ - global sa_logger, _IDLE_TIMEOUT, _MAX_RETRIES, _RETRY_INTERVAL, _CONNECTION - - _IDLE_TIMEOUT = conf.sql_idle_timeout - _MAX_RETRIES = conf.sql_max_retries - _RETRY_INTERVAL = conf.sql_retry_interval - _CONNECTION = conf.sql_connection - sa_logger = logging.getLogger('sqlalchemy.engine') - if conf.debug: - sa_logger.setLevel(logging.DEBUG) +log = logging.getLogger(__name__) def get_session(autocommit=True, expire_on_commit=False): - """Helper method to grab session""" - global _MAKER - if not _MAKER: - get_engine() - _get_maker(autocommit, expire_on_commit) - assert _MAKER - session = _MAKER() - return session - - -def get_engine(): - """Return a SQLAlchemy engine.""" - """May assign _ENGINE if not already assigned""" - global _ENGINE, sa_logger, _CONNECTION, _IDLE_TIMEOUT, _MAX_RETRIES,\ - _RETRY_INTERVAL - - if not _ENGINE: - setup_db_env() - - connection_dict = sqlalchemy.engine.url.make_url(_CONNECTION) - - engine_args = { - 'pool_recycle': _IDLE_TIMEOUT, - 'echo': False, - 'convert_unicode': True} - - try: - _ENGINE = sqlalchemy.create_engine(_CONNECTION, **engine_args) - - if 'mysql' in connection_dict.drivername: - sqlalchemy.event.listen(_ENGINE, 'checkout', _ping_listener) - - _ENGINE.connect = _wrap_db_error(_ENGINE.connect) - _ENGINE.connect() - except Exception as err: - msg = _("Error configuring registry database with supplied " - "sql_connection. Got error: %s") % err - log.error(msg) - raise - - if conf.db_auto_create: + if not session._MAKER: + if conf.database.auto_create: log.info(_('auto-creating DB')) _auto_create_db() else: log.info(_('not auto-creating DB')) - - return _ENGINE - - -def _get_maker(autocommit=True, expire_on_commit=False): - """Return a SQLAlchemy sessionmaker.""" - """May assign __MAKER if not already assigned""" - global _MAKER, _ENGINE - assert _ENGINE - if not _MAKER: - _MAKER = sessionmaker(bind=_ENGINE, autocommit=autocommit, - expire_on_commit=expire_on_commit) - return _MAKER - - -def _is_db_connection_error(args): - """Return True if error in connecting to db.""" - # NOTE(adam_g): This is currently MySQL specific and needs to be extended - # to support Postgres and others. - conn_err_codes = ('2002', '2003', '2006') - for err_code in conn_err_codes: - if args.find(err_code) != -1: - return True - return False - - -def _wrap_db_error(f): - """Retry DB connection. Copied from nova and modified.""" - def _wrap(*args, **kwargs): - try: - return f(*args, **kwargs) - except sqlalchemy.exc.OperationalError as e: - if not _is_db_connection_error(e.args[0]): - raise - - remaining_attempts = _MAX_RETRIES - while True: - log.warning(_('SQL connection failed. %d attempts left.'), - remaining_attempts) - remaining_attempts -= 1 - time.sleep(_RETRY_INTERVAL) - try: - return f(*args, **kwargs) - except sqlalchemy.exc.OperationalError as e: - if (remaining_attempts == 0 or - not _is_db_connection_error(e.args[0])): - raise - except sqlalchemy.exc.DBAPIError: - raise - except sqlalchemy.exc.DBAPIError: - raise - _wrap.func_name = f.func_name - return _wrap + return session.get_session(autocommit, expire_on_commit) def _auto_create_db(): repo_path = os.path.abspath(os.path.dirname(migrate_repo.__file__)) try: - versioning_api.upgrade(conf.sql_connection, repo_path) + versioning_api.upgrade(conf.database.connection, repo_path) except versioning_exceptions.DatabaseNotControlledError: - versioning_api.version_control(conf.sql_connection, repo_path) - versioning_api.upgrade(conf.sql_connection, repo_path) + versioning_api.version_control(conf.database.connection, repo_path) + versioning_api.upgrade(conf.database.connection, repo_path) diff --git a/muranoapi/openstack/common/log.py b/muranoapi/openstack/common/log.py index 74c5c005..c7c59b48 100644 --- a/muranoapi/openstack/common/log.py +++ b/muranoapi/openstack/common/log.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. diff --git a/muranoapi/openstack/common/timeutils.py b/muranoapi/openstack/common/timeutils.py index 21da74f4..ac2441bc 100644 --- a/muranoapi/openstack/common/timeutils.py +++ b/muranoapi/openstack/common/timeutils.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -66,7 +66,7 @@ def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT): def normalize_time(timestamp): - """Normalize time in arbitrary timezone to UTC naive object""" + """Normalize time in arbitrary timezone to UTC naive object.""" offset = timestamp.utcoffset() if offset is None: return timestamp @@ -103,7 +103,7 @@ def utcnow(): def iso8601_from_timestamp(timestamp): - """Returns a iso8601 formated date from timestamp""" + """Returns a iso8601 formated date from timestamp.""" return isotime(datetime.datetime.utcfromtimestamp(timestamp)) @@ -111,9 +111,9 @@ utcnow.override_time = None def set_time_override(override_time=datetime.datetime.utcnow()): - """ - Override utils.utcnow to return a constant time or a list thereof, - one at a time. + """Overrides utils.utcnow. + + Make it return a constant time or a list thereof, one at a time. """ utcnow.override_time = override_time @@ -141,7 +141,8 @@ def clear_time_override(): def marshall_now(now=None): """Make an rpc-safe datetime with microseconds. - Note: tzinfo is stripped, but not required for relative times.""" + Note: tzinfo is stripped, but not required for relative times. + """ if not now: now = utcnow() return dict(day=now.day, month=now.month, year=now.year, hour=now.hour, @@ -161,7 +162,8 @@ def unmarshall_time(tyme): def delta_seconds(before, after): - """ + """Return the difference between two timing objects. + Compute the difference in seconds between two date, time, or datetime objects (as a float, to microsecond resolution). """ @@ -174,8 +176,7 @@ def delta_seconds(before, after): def is_soon(dt, window): - """ - Determines if time is going to happen in the next window seconds. + """Determines if time is going to happen in the next window seconds. :params dt: the time :params window: minimum seconds to remain to consider the time not soon