From 4582a1a0dd2a4d515ac08957889a0d93cf750ab4 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Wed, 28 Aug 2013 15:49:17 +0400 Subject: [PATCH] Migrate to Oslo DB 96d1f887dda Part 3 Oslo version 96d1f887dda21b43ba4376187f31953dee6f5273 This commit just migrates Heat to new db related code from Oslo Partially implements blueprint oslo-db-support Change-Id: Ib0de40a70857fb3e029a2282f06746269a628c71 --- bin/heat-engine | 2 - .../heat/tests/test_rackspace_cloud_server.py | 3 +- etc/heat/heat.conf.sample | 8 -- heat/cmd/manage.py | 2 - heat/common/config.py | 10 -- heat/db/api.py | 21 +--- heat/db/migration.py | 8 +- heat/db/sqlalchemy/api.py | 16 ++- heat/db/sqlalchemy/migration.py | 8 +- heat/db/sqlalchemy/models.py | 63 +---------- heat/db/sqlalchemy/session.py | 102 ------------------ heat/db/sync.py | 3 - heat/tests/test_cw_alarm.py | 6 +- heat/tests/test_engine_service.py | 3 + heat/tests/test_resource.py | 2 +- heat/tests/test_sqlalchemy_api.py | 2 +- heat/tests/utils.py | 8 +- 17 files changed, 52 insertions(+), 215 deletions(-) delete mode 100644 heat/db/sqlalchemy/session.py diff --git a/bin/heat-engine b/bin/heat-engine index 483291c147..6f05e167c1 100755 --- a/bin/heat-engine +++ b/bin/heat-engine @@ -43,7 +43,6 @@ from oslo.config import cfg from heat.openstack.common import log as logging from heat.openstack.common import service -from heat.db import api as db_api from heat.rpc import api as rpc_api @@ -62,7 +61,6 @@ if __name__ == '__main__': from heat.engine import service as engine - db_api.configure() srv = engine.EngineService(cfg.CONF.host, rpc_api.ENGINE_TOPIC) launcher = service.launch(srv) launcher.wait() diff --git a/contrib/rackspace/heat/tests/test_rackspace_cloud_server.py b/contrib/rackspace/heat/tests/test_rackspace_cloud_server.py index aefae35c41..90e42c9d1d 100644 --- a/contrib/rackspace/heat/tests/test_rackspace_cloud_server.py +++ b/contrib/rackspace/heat/tests/test_rackspace_cloud_server.py @@ -434,7 +434,8 @@ class RackspaceCloudServerTest(HeatTestCase): cs._store_or_update(cs.CREATE, cs.IN_PROGRESS, 'test_store') cs.private_key = 'fake private key' - rs = db_api.resource_get_by_name_and_stack(None, + self.ctx = utils.dummy_context() + rs = db_api.resource_get_by_name_and_stack(self.ctx, 'cs_private_key', stack.id) encrypted_key = rs.data[0]['value'] diff --git a/etc/heat/heat.conf.sample b/etc/heat/heat.conf.sample index 1444f9ba2f..14bab00c79 100644 --- a/etc/heat/heat.conf.sample +++ b/etc/heat/heat.conf.sample @@ -4,14 +4,6 @@ # Options defined in heat.common.config # -# The SQLAlchemy connection string used to connect to the -# database (string value) -#sql_connection=mysql://heat:heat@localhost/heat - -# timeout before idle sql connections are reaped (integer -# value) -#sql_idle_timeout=3600 - # The default user for new instances (string value) #instance_user=ec2-user diff --git a/heat/cmd/manage.py b/heat/cmd/manage.py index d3852fb4b5..1d1ed0d4c6 100644 --- a/heat/cmd/manage.py +++ b/heat/cmd/manage.py @@ -22,7 +22,6 @@ import sys from oslo.config import cfg -from heat.db import api as db_api from heat.db import migration from heat.db import utils from heat.openstack.common import log @@ -79,7 +78,6 @@ def main(): version=version.version_info.version_string(), default_config_files=default_config_files) log.setup("heat") - db_api.configure() except RuntimeError as e: sys.exit("ERROR: %s" % e) diff --git a/heat/common/config.py b/heat/common/config.py index 82b4ca5db8..3eea73e133 100644 --- a/heat/common/config.py +++ b/heat/common/config.py @@ -68,15 +68,6 @@ service_opts = [ default=3, help='Maximum depth allowed when using nested stacks.')] -db_opts = [ - cfg.StrOpt('sql_connection', - default='mysql://heat:heat@localhost/heat', - help='The SQLAlchemy connection string used to connect to the ' - 'database'), - cfg.IntOpt('sql_idle_timeout', - default=3600, - help='timeout before idle sql connections are reaped')] - engine_opts = [ cfg.StrOpt('instance_user', default='ec2-user', @@ -134,7 +125,6 @@ auth_password_opts = [ 'multi_cloud is enabled. At least one endpoint needs ' 'to be specified.'))] -cfg.CONF.register_opts(db_opts) cfg.CONF.register_opts(engine_opts) cfg.CONF.register_opts(service_opts) cfg.CONF.register_opts(rpc_opts) diff --git a/heat/db/api.py b/heat/db/api.py index 2810a66592..b9bc0c268b 100644 --- a/heat/db/api.py +++ b/heat/db/api.py @@ -28,30 +28,19 @@ supported backend. from oslo.config import cfg -from heat.db import utils +from heat.openstack.common.db import api as db_api -SQL_CONNECTION = 'sqlite://' -SQL_IDLE_TIMEOUT = 3600 db_opts = [ cfg.StrOpt('db_backend', default='sqlalchemy', help='The backend to use for db')] -cfg.CONF.register_opts(db_opts) +CONF = cfg.CONF +CONF.register_opts(db_opts) -IMPL = utils.LazyPluggable('db_backend', - sqlalchemy='heat.db.sqlalchemy.api') +_BACKEND_MAPPING = {'sqlalchemy': 'heat.db.sqlalchemy.api'} - -cfg.CONF.import_opt('sql_connection', 'heat.common.config') -cfg.CONF.import_opt('sql_idle_timeout', 'heat.common.config') - - -def configure(): - global SQL_CONNECTION - global SQL_IDLE_TIMEOUT - SQL_CONNECTION = cfg.CONF.sql_connection - SQL_IDLE_TIMEOUT = cfg.CONF.sql_idle_timeout +IMPL = db_api.DBAPI(backend_mapping=_BACKEND_MAPPING) def get_session(): diff --git a/heat/db/migration.py b/heat/db/migration.py index 73c9cfd9dd..4f6a7507cd 100644 --- a/heat/db/migration.py +++ b/heat/db/migration.py @@ -14,13 +14,13 @@ """Database setup and migration commands.""" -from heat.db import utils +from heat.openstack.common.db import api as db_api -IMPL = utils.LazyPluggable('db_backend', - sqlalchemy='heat.db.sqlalchemy.migration') - INIT_VERSION = 14 +_BACKEND_MAPPING = {'sqlalchemy': 'heat.db.sqlalchemy.migration'} + +IMPL = db_api.DBAPI(backend_mapping=_BACKEND_MAPPING) def db_sync(version=None): diff --git a/heat/db/sqlalchemy/api.py b/heat/db/sqlalchemy/api.py index 426879eaa4..1d36a24851 100644 --- a/heat/db/sqlalchemy/api.py +++ b/heat/db/sqlalchemy/api.py @@ -14,6 +14,7 @@ # under the License. '''Implementation of SQLAlchemy backend.''' +import sys from datetime import datetime from datetime import timedelta @@ -28,8 +29,17 @@ from heat.openstack.common.gettextutils import _ from heat.common import crypt from heat.common import exception from heat.db.sqlalchemy import models -from heat.db.sqlalchemy.session import get_engine -from heat.db.sqlalchemy.session import get_session +from heat.openstack.common.db.sqlalchemy import session as db_session + + +get_engine = db_session.get_engine +get_session = db_session.get_session + + +def get_backend(): + """The backend is this module itself.""" + + return sys.modules[__name__] def model_query(context, *args): @@ -160,7 +170,7 @@ def resource_data_set(resource, key, value, redact=False): current.resource_id = resource.id current.redact = redact current.value = value - current.save() + current.save(session=resource.context.session) return current diff --git a/heat/db/sqlalchemy/migration.py b/heat/db/sqlalchemy/migration.py index eeaa255b95..fa0782f5ad 100644 --- a/heat/db/sqlalchemy/migration.py +++ b/heat/db/sqlalchemy/migration.py @@ -16,7 +16,6 @@ import distutils.version as dist_version import os import sys -from heat.db.sqlalchemy.session import get_engine from heat.db import migration import sqlalchemy @@ -24,11 +23,18 @@ import migrate from migrate.versioning import util as migrate_util from heat.openstack.common import exception +from heat.openstack.common.db.sqlalchemy.session import get_engine from heat.openstack.common.gettextutils import _ _REPOSITORY = None +def get_backend(): + """The backend is this module itself.""" + + return sys.modules[__name__] + + @migrate_util.decorator def patched_with_engine(f, *a, **kw): url = a[0] diff --git a/heat/db/sqlalchemy/models.py b/heat/db/sqlalchemy/models.py index 535f01d582..5ded70b1ba 100644 --- a/heat/db/sqlalchemy/models.py +++ b/heat/db/sqlalchemy/models.py @@ -18,19 +18,19 @@ SQLAlchemy models for heat data. import sqlalchemy from sqlalchemy.dialects import mysql -from sqlalchemy.orm import relationship, backref, object_mapper -from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import relationship, backref from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import types from json import dumps from json import loads -from heat.openstack.common import exception from heat.openstack.common import uuidutils from heat.openstack.common import timeutils -from heat.db.sqlalchemy.session import get_session +from heat.openstack.common.db.sqlalchemy import models +from heat.openstack.common.db.sqlalchemy import session from sqlalchemy.orm.session import Session BASE = declarative_base() +get_session = session.get_session class Json(types.TypeDecorator): @@ -58,29 +58,9 @@ except ImportError: MutableDict.associate_with(Json) -class HeatBase(object): +class HeatBase(models.ModelBase, models.TimestampMixin): """Base class for Heat Models.""" __table_args__ = {'mysql_engine': 'InnoDB'} - __table_initialized__ = False - created_at = sqlalchemy.Column(sqlalchemy.DateTime, - default=timeutils.utcnow) - updated_at = sqlalchemy.Column(sqlalchemy.DateTime, - onupdate=timeutils.utcnow) - - def save(self, session=None): - """Save this object.""" - if not session: - session = Session.object_session(self) - if not session: - session = get_session() - session.add(self) - try: - session.flush() - except IntegrityError as e: - if str(e).endswith('is not unique'): - raise exception.Duplicate(str(e)) - else: - raise def expire(self, session=None, attrs=None): """Expire this object ().""" @@ -107,28 +87,6 @@ class HeatBase(object): session.delete(self) session.flush() - def __setitem__(self, key, value): - setattr(self, key, value) - - def __getitem__(self, key): - return getattr(self, key) - - def get(self, key, default=None): - return getattr(self, key, default) - - def __iter__(self): - self._i = iter(object_mapper(self).columns) - return self - - def next(self): - n = self._i.next().name - return n, getattr(self, n) - - def update(self, values): - """Make the model object behave like a dict.""" - for k, v in values.iteritems(): - setattr(self, k, v) - def update_and_save(self, values, session=None): if not session: session = Session.object_session(self) @@ -139,17 +97,6 @@ class HeatBase(object): setattr(self, k, v) session.commit() - def iteritems(self): - """Make the model object behave like a dict. - - Includes attributes from joins. - """ - local = dict(self) - joined = dict([(k, v) for k, v in self.__dict__.iteritems() - if not k[0] == '_']) - local.update(joined) - return local.iteritems() - class SoftDelete(object): deleted_at = sqlalchemy.Column(sqlalchemy.DateTime) diff --git a/heat/db/sqlalchemy/session.py b/heat/db/sqlalchemy/session.py deleted file mode 100644 index 72daff4723..0000000000 --- a/heat/db/sqlalchemy/session.py +++ /dev/null @@ -1,102 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Session Handling for SQLAlchemy backend.""" - -import sqlalchemy.interfaces -import sqlalchemy.orm -import sqlalchemy.engine -from sqlalchemy.exc import DisconnectionError - -from heat.openstack.common import log as logging - -from heat.db import api as db_api - -logger = logging.getLogger(__name__) -_ENGINE = None -_MAKER = None - - -def get_session(autocommit=True, expire_on_commit=False): - """Return a SQLAlchemy session.""" - global _MAKER - - if _MAKER is None: - _MAKER = get_maker(get_engine(), autocommit, expire_on_commit) - return _MAKER() - - -class SynchronousSwitchListener(sqlalchemy.interfaces.PoolListener): - - """Switch sqlite connections to non-synchronous mode.""" - - def connect(self, dbapi_con, con_record): - dbapi_con.execute("PRAGMA synchronous = OFF") - - -class MySQLPingListener(object): - - """ - Ensures that MySQL connections checked out of the - pool are alive. - - Borrowed from: - http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f - """ - - def checkout(self, dbapi_con, con_record, con_proxy): - try: - dbapi_con.cursor().execute('select 1') - except dbapi_con.OperationalError as ex: - if ex.args[0] in (2006, 2013, 2014, 2045, 2055): - logger.warn('Got mysql server has gone away: %s', ex) - raise DisconnectionError("Database server went away") - else: - raise - - -def get_engine(): - """Return a SQLAlchemy engine.""" - global _ENGINE - if _ENGINE is None: - connection_dict = sqlalchemy.engine.url.make_url(_get_sql_connection()) - engine_args = { - "pool_recycle": _get_sql_idle_timeout(), - "echo": False, - 'convert_unicode': True - } - - if 'mysql' in connection_dict.drivername: - engine_args['listeners'] = [MySQLPingListener()] - - _ENGINE = sqlalchemy.create_engine(_get_sql_connection(), - **engine_args) - return _ENGINE - - -def get_maker(engine, autocommit=True, expire_on_commit=False): - """Return a SQLAlchemy sessionmaker using the given engine.""" - ses = sqlalchemy.orm.sessionmaker( - bind=engine, - autocommit=autocommit, - expire_on_commit=expire_on_commit) - return sqlalchemy.orm.scoped_session(ses) - - -def _get_sql_connection(): - return db_api.SQL_CONNECTION - - -def _get_sql_idle_timeout(): - return db_api.SQL_IDLE_TIMEOUT diff --git a/heat/db/sync.py b/heat/db/sync.py index 9eac77992a..0db39e3c28 100755 --- a/heat/db/sync.py +++ b/heat/db/sync.py @@ -24,7 +24,6 @@ gettextutils.install('heat') from oslo.config import cfg from heat.openstack.common import log as logging -from heat.db import api from heat.db import migration LOG = logging.getLogger(__name__) @@ -36,8 +35,6 @@ if __name__ == '__main__': print('*******************************************', file=sys.stderr) cfg.CONF(project='heat', prog='heat-engine') - api.configure() - try: migration.db_sync() except Exception as exc: diff --git a/heat/tests/test_cw_alarm.py b/heat/tests/test_cw_alarm.py index 8729ef669e..d09fb76fef 100644 --- a/heat/tests/test_cw_alarm.py +++ b/heat/tests/test_cw_alarm.py @@ -133,8 +133,10 @@ class CloudWatchAlarmTest(HeatTestCase): scheduler.TaskRunner(rsrc.suspend)() self.assertEqual(rsrc.state, (rsrc.SUSPEND, rsrc.COMPLETE)) + self.ctx = utils.dummy_context() + wr = watchrule.WatchRule.load( - None, watch_name="test_stack-MEMAlarmHigh") + self.ctx, watch_name="test_stack-MEMAlarmHigh") self.assertEqual(wr.state, watchrule.WatchRule.SUSPENDED) @@ -142,7 +144,7 @@ class CloudWatchAlarmTest(HeatTestCase): self.assertEqual(rsrc.state, (rsrc.RESUME, rsrc.COMPLETE)) wr = watchrule.WatchRule.load( - None, watch_name="test_stack-MEMAlarmHigh") + self.ctx, watch_name="test_stack-MEMAlarmHigh") self.assertEqual(wr.state, watchrule.WatchRule.NODATA) diff --git a/heat/tests/test_engine_service.py b/heat/tests/test_engine_service.py index f6a104406c..43038470bc 100644 --- a/heat/tests/test_engine_service.py +++ b/heat/tests/test_engine_service.py @@ -718,6 +718,9 @@ class StackServiceCreateUpdateDeleteTest(HeatTestCase): self.m.StubOutWithMock(parser.Stack, 'load') self.m.StubOutWithMock(parser, 'Template') self.m.StubOutWithMock(environment, 'Environment') + self.m.StubOutWithMock(self.man, '_get_stack') + + self.man._get_stack(self.ctx, old_stack.identifier()).AndReturn(s) parser.Stack.load(self.ctx, stack=s).AndReturn(old_stack) diff --git a/heat/tests/test_resource.py b/heat/tests/test_resource.py index 588c6f5ac9..5b447ddc16 100644 --- a/heat/tests/test_resource.py +++ b/heat/tests/test_resource.py @@ -137,7 +137,7 @@ class ResourceTest(HeatTestCase): self.assertEqual(res.status, res.IN_PROGRESS) self.assertEqual(res.status_reason, 'test_store') - db_res = r = db_api.resource_get(None, res.id) + db_res = r = db_api.resource_get(res.context, res.id) self.assertEqual(db_res.action, res.CREATE) self.assertEqual(db_res.status, res.IN_PROGRESS) self.assertEqual(db_res.status_reason, 'test_store') diff --git a/heat/tests/test_sqlalchemy_api.py b/heat/tests/test_sqlalchemy_api.py index 5032a9949e..ebec0c7c8e 100644 --- a/heat/tests/test_sqlalchemy_api.py +++ b/heat/tests/test_sqlalchemy_api.py @@ -130,7 +130,7 @@ class SqlAlchemyTest(HeatTestCase): cs._store_or_update(cs.CREATE, cs.IN_PROGRESS, 'test_store') cs.my_secret = 'fake secret' - rs = db_api.resource_get_by_name_and_stack(None, + rs = db_api.resource_get_by_name_and_stack(self.ctx, 'cs_encryption', stack.id) encrypted_key = rs.data[0]['value'] diff --git a/heat/tests/utils.py b/heat/tests/utils.py index 6aa2db0a81..448862a7d1 100644 --- a/heat/tests/utils.py +++ b/heat/tests/utils.py @@ -20,13 +20,17 @@ import uuid import sqlalchemy +from oslo.config import cfg + from heat.common import context from heat.common import exception from heat.engine import environment from heat.engine import parser -from heat.db.sqlalchemy.session import get_engine from heat.db import migration +from heat.openstack.common.db.sqlalchemy import session + +get_engine = session.get_engine class UUIDStub(object): @@ -112,6 +116,8 @@ def wr_delete_after(test_fn): def setup_dummy_db(): + cfg.CONF.set_default('sqlite_synchronous', False) + session.set_defaults(sql_connection="sqlite://", sqlite_db='heat.db') migration.db_sync() engine = get_engine() engine.connect()