Merge "Migrate to Oslo DB 96d1f887dda Part 3"

bin/heat-engine

@@ -43,7 +43,6 @@ from oslo.config import cfg
 from heat.openstack.common import log as logging
 from heat.openstack.common import service
 
-from heat.db import api as db_api
 from heat.rpc import api as rpc_api
 
 
@@ -62,7 +61,6 @@ if __name__ == '__main__':
 
     from heat.engine import service as engine
 
-    db_api.configure()
     srv = engine.EngineService(cfg.CONF.host, rpc_api.ENGINE_TOPIC)
     launcher = service.launch(srv)
     launcher.wait()

contrib/rackspace/heat/tests/test_rackspace_cloud_server.py

@@ -470,7 +470,8 @@ class RackspaceCloudServerTest(HeatTestCase):
         cs._store_or_update(cs.CREATE, cs.IN_PROGRESS, 'test_store')
 
         cs.private_key = 'fake private key'
-        rs = db_api.resource_get_by_name_and_stack(None,
+        self.ctx = utils.dummy_context()
+        rs = db_api.resource_get_by_name_and_stack(self.ctx,
                                                    'cs_private_key',
                                                    stack.id)
         encrypted_key = rs.data[0]['value']

etc/heat/heat.conf.sample

@@ -4,14 +4,6 @@
 # Options defined in heat.common.config
 #
 
-# The SQLAlchemy connection string used to connect to the
-# database (string value)
-#sql_connection=mysql://heat:heat@localhost/heat
-
-# timeout before idle sql connections are reaped (integer
-# value)
-#sql_idle_timeout=3600
-
 # The default user for new instances (string value)
 #instance_user=ec2-user
 

heat/cmd/manage.py

@@ -22,7 +22,6 @@ import sys
 
 from oslo.config import cfg
 
-from heat.db import api as db_api
 from heat.db import migration
 from heat.db import utils
 from heat.openstack.common import log
@@ -80,7 +79,6 @@ def main():
              version=version.version_info.version_string(),
              default_config_files=default_config_files)
         log.setup("heat")
-        db_api.configure()
     except RuntimeError as e:
         sys.exit("ERROR: %s" % e)
 

heat/common/config.py

@@ -68,15 +68,6 @@ service_opts = [
                default=3,
                help='Maximum depth allowed when using nested stacks.')]
 
-db_opts = [
-    cfg.StrOpt('sql_connection',
-               default='mysql://heat:heat@localhost/heat',
-               help='The SQLAlchemy connection string used to connect to the '
-               'database'),
-    cfg.IntOpt('sql_idle_timeout',
-               default=3600,
-               help='timeout before idle sql connections are reaped')]
-
 engine_opts = [
     cfg.StrOpt('instance_user',
                default='ec2-user',
@@ -134,7 +125,6 @@ auth_password_opts = [
                        'multi_cloud is enabled. At least one endpoint needs '
                        'to be specified.'))]
 
-cfg.CONF.register_opts(db_opts)
 cfg.CONF.register_opts(engine_opts)
 cfg.CONF.register_opts(service_opts)
 cfg.CONF.register_opts(rpc_opts)

heat/db/api.py

@@ -28,30 +28,19 @@ supported backend.
 
 from oslo.config import cfg
 
-from heat.db import utils
+from heat.openstack.common.db import api as db_api
 
-SQL_CONNECTION = 'sqlite://'
-SQL_IDLE_TIMEOUT = 3600
 db_opts = [
     cfg.StrOpt('db_backend',
                default='sqlalchemy',
                help='The backend to use for db')]
 
-cfg.CONF.register_opts(db_opts)
+CONF = cfg.CONF
+CONF.register_opts(db_opts)
 
-IMPL = utils.LazyPluggable('db_backend',
-                           sqlalchemy='heat.db.sqlalchemy.api')
+_BACKEND_MAPPING = {'sqlalchemy': 'heat.db.sqlalchemy.api'}
 
-
-cfg.CONF.import_opt('sql_connection', 'heat.common.config')
-cfg.CONF.import_opt('sql_idle_timeout', 'heat.common.config')
-
-
-def configure():
-    global SQL_CONNECTION
-    global SQL_IDLE_TIMEOUT
-    SQL_CONNECTION = cfg.CONF.sql_connection
-    SQL_IDLE_TIMEOUT = cfg.CONF.sql_idle_timeout
+IMPL = db_api.DBAPI(backend_mapping=_BACKEND_MAPPING)
 
 
 def get_session():

heat/db/migration.py

@@ -14,13 +14,13 @@
 
 """Database setup and migration commands."""
 
-from heat.db import utils
+from heat.openstack.common.db import api as db_api
 
 
-IMPL = utils.LazyPluggable('db_backend',
-                           sqlalchemy='heat.db.sqlalchemy.migration')
-
 INIT_VERSION = 14
+_BACKEND_MAPPING = {'sqlalchemy': 'heat.db.sqlalchemy.migration'}
+
+IMPL = db_api.DBAPI(backend_mapping=_BACKEND_MAPPING)
 
 
 def db_sync(version=None):

heat/db/sqlalchemy/api.py

@@ -14,6 +14,7 @@
 #    under the License.
 
 '''Implementation of SQLAlchemy backend.'''
+import sys
 from datetime import datetime
 from datetime import timedelta
 
@@ -28,8 +29,17 @@ from heat.openstack.common.gettextutils import _
 from heat.common import crypt
 from heat.common import exception
 from heat.db.sqlalchemy import models
-from heat.db.sqlalchemy.session import get_engine
-from heat.db.sqlalchemy.session import get_session
+from heat.openstack.common.db.sqlalchemy import session as db_session
+
+
+get_engine = db_session.get_engine
+get_session = db_session.get_session
+
+
+def get_backend():
+    """The backend is this module itself."""
+
+    return sys.modules[__name__]
 
 
 def model_query(context, *args):
@@ -161,7 +171,7 @@ def resource_data_set(resource, key, value, redact=False):
         current.resource_id = resource.id
     current.redact = redact
     current.value = value
-    current.save()
+    current.save(session=resource.context.session)
     return current
 
 

heat/db/sqlalchemy/migration.py

@@ -16,7 +16,6 @@ import distutils.version as dist_version
 import os
 import sys
 
-from heat.db.sqlalchemy.session import get_engine
 from heat.db import migration
 
 import sqlalchemy
@@ -24,11 +23,18 @@ import migrate
 from migrate.versioning import util as migrate_util
 
 from heat.openstack.common import exception
+from heat.openstack.common.db.sqlalchemy.session import get_engine
 from heat.openstack.common.gettextutils import _
 
 _REPOSITORY = None
 
 
+def get_backend():
+    """The backend is this module itself."""
+
+    return sys.modules[__name__]
+
+
 @migrate_util.decorator
 def patched_with_engine(f, *a, **kw):
     url = a[0]

heat/db/sqlalchemy/models.py

@@ -18,19 +18,19 @@ SQLAlchemy models for heat data.
 import sqlalchemy
 
 from sqlalchemy.dialects import mysql
-from sqlalchemy.orm import relationship, backref, object_mapper
-from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import relationship, backref
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import types
 from json import dumps
 from json import loads
-from heat.openstack.common import exception
 from heat.openstack.common import uuidutils
 from heat.openstack.common import timeutils
-from heat.db.sqlalchemy.session import get_session
+from heat.openstack.common.db.sqlalchemy import models
+from heat.openstack.common.db.sqlalchemy import session
 from sqlalchemy.orm.session import Session
 
 BASE = declarative_base()
+get_session = session.get_session
 
 
 class Json(types.TypeDecorator):
@@ -58,29 +58,9 @@ except ImportError:
     MutableDict.associate_with(Json)
 
 
-class HeatBase(object):
+class HeatBase(models.ModelBase, models.TimestampMixin):
     """Base class for Heat Models."""
     __table_args__ = {'mysql_engine': 'InnoDB'}
-    __table_initialized__ = False
-    created_at = sqlalchemy.Column(sqlalchemy.DateTime,
-                                   default=timeutils.utcnow)
-    updated_at = sqlalchemy.Column(sqlalchemy.DateTime,
-                                   onupdate=timeutils.utcnow)
-
-    def save(self, session=None):
-        """Save this object."""
-        if not session:
-            session = Session.object_session(self)
-            if not session:
-                session = get_session()
-        session.add(self)
-        try:
-            session.flush()
-        except IntegrityError as e:
-            if str(e).endswith('is not unique'):
-                raise exception.Duplicate(str(e))
-            else:
-                raise
 
     def expire(self, session=None, attrs=None):
         """Expire this object ()."""
@@ -107,28 +87,6 @@ class HeatBase(object):
         session.delete(self)
         session.flush()
 
-    def __setitem__(self, key, value):
-        setattr(self, key, value)
-
-    def __getitem__(self, key):
-        return getattr(self, key)
-
-    def get(self, key, default=None):
-        return getattr(self, key, default)
-
-    def __iter__(self):
-        self._i = iter(object_mapper(self).columns)
-        return self
-
-    def next(self):
-        n = self._i.next().name
-        return n, getattr(self, n)
-
-    def update(self, values):
-        """Make the model object behave like a dict."""
-        for k, v in values.iteritems():
-            setattr(self, k, v)
-
     def update_and_save(self, values, session=None):
         if not session:
             session = Session.object_session(self)
@@ -139,17 +97,6 @@ class HeatBase(object):
             setattr(self, k, v)
         session.commit()
 
-    def iteritems(self):
-        """Make the model object behave like a dict.
-
-        Includes attributes from joins.
-        """
-        local = dict(self)
-        joined = dict([(k, v) for k, v in self.__dict__.iteritems()
-                      if not k[0] == '_'])
-        local.update(joined)
-        return local.iteritems()
-
 
 class SoftDelete(object):
     deleted_at = sqlalchemy.Column(sqlalchemy.DateTime)

heat/db/sqlalchemy/session.py

@@ -1,102 +0,0 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-"""Session Handling for SQLAlchemy backend."""
-
-import sqlalchemy.interfaces
-import sqlalchemy.orm
-import sqlalchemy.engine
-from sqlalchemy.exc import DisconnectionError
-
-from heat.openstack.common import log as logging
-
-from heat.db import api as db_api
-
-logger = logging.getLogger(__name__)
-_ENGINE = None
-_MAKER = None
-
-
-def get_session(autocommit=True, expire_on_commit=False):
-    """Return a SQLAlchemy session."""
-    global _MAKER
-
-    if _MAKER is None:
-        _MAKER = get_maker(get_engine(), autocommit, expire_on_commit)
-    return _MAKER()
-
-
-class SynchronousSwitchListener(sqlalchemy.interfaces.PoolListener):
-
-    """Switch sqlite connections to non-synchronous mode."""
-
-    def connect(self, dbapi_con, con_record):
-        dbapi_con.execute("PRAGMA synchronous = OFF")
-
-
-class MySQLPingListener(object):
-
-    """
-    Ensures that MySQL connections checked out of the
-    pool are alive.
-
-    Borrowed from:
-    http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f
-    """
-
-    def checkout(self, dbapi_con, con_record, con_proxy):
-        try:
-            dbapi_con.cursor().execute('select 1')
-        except dbapi_con.OperationalError as ex:
-            if ex.args[0] in (2006, 2013, 2014, 2045, 2055):
-                logger.warn('Got mysql server has gone away: %s', ex)
-                raise DisconnectionError("Database server went away")
-            else:
-                raise
-
-
-def get_engine():
-    """Return a SQLAlchemy engine."""
-    global _ENGINE
-    if _ENGINE is None:
-        connection_dict = sqlalchemy.engine.url.make_url(_get_sql_connection())
-        engine_args = {
-            "pool_recycle": _get_sql_idle_timeout(),
-            "echo": False,
-            'convert_unicode': True
-        }
-
-        if 'mysql' in connection_dict.drivername:
-            engine_args['listeners'] = [MySQLPingListener()]
-
-        _ENGINE = sqlalchemy.create_engine(_get_sql_connection(),
-                                           **engine_args)
-    return _ENGINE
-
-
-def get_maker(engine, autocommit=True, expire_on_commit=False):
-    """Return a SQLAlchemy sessionmaker using the given engine."""
-    ses = sqlalchemy.orm.sessionmaker(
-        bind=engine,
-        autocommit=autocommit,
-        expire_on_commit=expire_on_commit)
-    return sqlalchemy.orm.scoped_session(ses)
-
-
-def _get_sql_connection():
-    return db_api.SQL_CONNECTION
-
-
-def _get_sql_idle_timeout():
-    return db_api.SQL_IDLE_TIMEOUT

heat/db/sync.py

@@ -24,7 +24,6 @@ gettextutils.install('heat')
 
 from oslo.config import cfg
 from heat.openstack.common import log as logging
-from heat.db import api
 from heat.db import migration
 
 LOG = logging.getLogger(__name__)
@@ -36,8 +35,6 @@ if __name__ == '__main__':
     print('*******************************************', file=sys.stderr)
     cfg.CONF(project='heat', prog='heat-engine')
 
-    api.configure()
-
     try:
         migration.db_sync()
     except Exception as exc:

heat/tests/test_cw_alarm.py

@@ -133,8 +133,10 @@ class CloudWatchAlarmTest(HeatTestCase):
         scheduler.TaskRunner(rsrc.suspend)()
         self.assertEqual(rsrc.state, (rsrc.SUSPEND, rsrc.COMPLETE))
 
+        self.ctx = utils.dummy_context()
+
         wr = watchrule.WatchRule.load(
-            None, watch_name="test_stack-MEMAlarmHigh")
+            self.ctx, watch_name="test_stack-MEMAlarmHigh")
 
         self.assertEqual(wr.state, watchrule.WatchRule.SUSPENDED)
 
@@ -142,7 +144,7 @@ class CloudWatchAlarmTest(HeatTestCase):
         self.assertEqual(rsrc.state, (rsrc.RESUME, rsrc.COMPLETE))
 
         wr = watchrule.WatchRule.load(
-            None, watch_name="test_stack-MEMAlarmHigh")
+            self.ctx, watch_name="test_stack-MEMAlarmHigh")
 
         self.assertEqual(wr.state, watchrule.WatchRule.NODATA)
 

heat/tests/test_engine_service.py

@@ -718,6 +718,9 @@ class StackServiceCreateUpdateDeleteTest(HeatTestCase):
         self.m.StubOutWithMock(parser.Stack, 'load')
         self.m.StubOutWithMock(parser, 'Template')
         self.m.StubOutWithMock(environment, 'Environment')
+        self.m.StubOutWithMock(self.man, '_get_stack')
+
+        self.man._get_stack(self.ctx, old_stack.identifier()).AndReturn(s)
 
         parser.Stack.load(self.ctx, stack=s).AndReturn(old_stack)
 

heat/tests/test_resource.py

@@ -137,7 +137,7 @@ class ResourceTest(HeatTestCase):
         self.assertEqual(res.status, res.IN_PROGRESS)
         self.assertEqual(res.status_reason, 'test_store')
 
-        db_res = r = db_api.resource_get(None, res.id)
+        db_res = r = db_api.resource_get(res.context, res.id)
         self.assertEqual(db_res.action, res.CREATE)
         self.assertEqual(db_res.status, res.IN_PROGRESS)
         self.assertEqual(db_res.status_reason, 'test_store')

heat/tests/test_sqlalchemy_api.py

@@ -130,7 +130,7 @@ class SqlAlchemyTest(HeatTestCase):
         cs._store_or_update(cs.CREATE, cs.IN_PROGRESS, 'test_store')
 
         cs.my_secret = 'fake secret'
-        rs = db_api.resource_get_by_name_and_stack(None,
+        rs = db_api.resource_get_by_name_and_stack(self.ctx,
                                                    'cs_encryption',
                                                    stack.id)
         encrypted_key = rs.data[0]['value']

heat/tests/utils.py

@@ -20,13 +20,17 @@ import uuid
 
 import sqlalchemy
 
+from oslo.config import cfg
+
 from heat.common import context
 from heat.common import exception
 from heat.engine import environment
 from heat.engine import parser
 
-from heat.db.sqlalchemy.session import get_engine
 from heat.db import migration
+from heat.openstack.common.db.sqlalchemy import session
+
+get_engine = session.get_engine
 
 
 class UUIDStub(object):
@@ -112,6 +116,8 @@ def wr_delete_after(test_fn):
 
 
 def setup_dummy_db():
+    cfg.CONF.set_default('sqlite_synchronous', False)
+    session.set_defaults(sql_connection="sqlite://", sqlite_db='heat.db')
     migration.db_sync()
     engine = get_engine()
     engine.connect()