Migrate to Oslo DB 96d1f887dda Part 3
Oslo version 96d1f887dda21b43ba4376187f31953dee6f5273 This commit just migrates Heat to new db related code from Oslo Partially implements blueprint oslo-db-support Change-Id: Ib0de40a70857fb3e029a2282f06746269a628c71
This commit is contained in:
parent
06a6609035
commit
4582a1a0dd
@ -43,7 +43,6 @@ from oslo.config import cfg
|
||||
from heat.openstack.common import log as logging
|
||||
from heat.openstack.common import service
|
||||
|
||||
from heat.db import api as db_api
|
||||
from heat.rpc import api as rpc_api
|
||||
|
||||
|
||||
@ -62,7 +61,6 @@ if __name__ == '__main__':
|
||||
|
||||
from heat.engine import service as engine
|
||||
|
||||
db_api.configure()
|
||||
srv = engine.EngineService(cfg.CONF.host, rpc_api.ENGINE_TOPIC)
|
||||
launcher = service.launch(srv)
|
||||
launcher.wait()
|
||||
|
@ -434,7 +434,8 @@ class RackspaceCloudServerTest(HeatTestCase):
|
||||
cs._store_or_update(cs.CREATE, cs.IN_PROGRESS, 'test_store')
|
||||
|
||||
cs.private_key = 'fake private key'
|
||||
rs = db_api.resource_get_by_name_and_stack(None,
|
||||
self.ctx = utils.dummy_context()
|
||||
rs = db_api.resource_get_by_name_and_stack(self.ctx,
|
||||
'cs_private_key',
|
||||
stack.id)
|
||||
encrypted_key = rs.data[0]['value']
|
||||
|
@ -4,14 +4,6 @@
|
||||
# Options defined in heat.common.config
|
||||
#
|
||||
|
||||
# The SQLAlchemy connection string used to connect to the
|
||||
# database (string value)
|
||||
#sql_connection=mysql://heat:heat@localhost/heat
|
||||
|
||||
# timeout before idle sql connections are reaped (integer
|
||||
# value)
|
||||
#sql_idle_timeout=3600
|
||||
|
||||
# The default user for new instances (string value)
|
||||
#instance_user=ec2-user
|
||||
|
||||
|
@ -22,7 +22,6 @@ import sys
|
||||
|
||||
from oslo.config import cfg
|
||||
|
||||
from heat.db import api as db_api
|
||||
from heat.db import migration
|
||||
from heat.db import utils
|
||||
from heat.openstack.common import log
|
||||
@ -79,7 +78,6 @@ def main():
|
||||
version=version.version_info.version_string(),
|
||||
default_config_files=default_config_files)
|
||||
log.setup("heat")
|
||||
db_api.configure()
|
||||
except RuntimeError as e:
|
||||
sys.exit("ERROR: %s" % e)
|
||||
|
||||
|
@ -68,15 +68,6 @@ service_opts = [
|
||||
default=3,
|
||||
help='Maximum depth allowed when using nested stacks.')]
|
||||
|
||||
db_opts = [
|
||||
cfg.StrOpt('sql_connection',
|
||||
default='mysql://heat:heat@localhost/heat',
|
||||
help='The SQLAlchemy connection string used to connect to the '
|
||||
'database'),
|
||||
cfg.IntOpt('sql_idle_timeout',
|
||||
default=3600,
|
||||
help='timeout before idle sql connections are reaped')]
|
||||
|
||||
engine_opts = [
|
||||
cfg.StrOpt('instance_user',
|
||||
default='ec2-user',
|
||||
@ -134,7 +125,6 @@ auth_password_opts = [
|
||||
'multi_cloud is enabled. At least one endpoint needs '
|
||||
'to be specified.'))]
|
||||
|
||||
cfg.CONF.register_opts(db_opts)
|
||||
cfg.CONF.register_opts(engine_opts)
|
||||
cfg.CONF.register_opts(service_opts)
|
||||
cfg.CONF.register_opts(rpc_opts)
|
||||
|
@ -28,30 +28,19 @@ supported backend.
|
||||
|
||||
from oslo.config import cfg
|
||||
|
||||
from heat.db import utils
|
||||
from heat.openstack.common.db import api as db_api
|
||||
|
||||
SQL_CONNECTION = 'sqlite://'
|
||||
SQL_IDLE_TIMEOUT = 3600
|
||||
db_opts = [
|
||||
cfg.StrOpt('db_backend',
|
||||
default='sqlalchemy',
|
||||
help='The backend to use for db')]
|
||||
|
||||
cfg.CONF.register_opts(db_opts)
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opts(db_opts)
|
||||
|
||||
IMPL = utils.LazyPluggable('db_backend',
|
||||
sqlalchemy='heat.db.sqlalchemy.api')
|
||||
_BACKEND_MAPPING = {'sqlalchemy': 'heat.db.sqlalchemy.api'}
|
||||
|
||||
|
||||
cfg.CONF.import_opt('sql_connection', 'heat.common.config')
|
||||
cfg.CONF.import_opt('sql_idle_timeout', 'heat.common.config')
|
||||
|
||||
|
||||
def configure():
|
||||
global SQL_CONNECTION
|
||||
global SQL_IDLE_TIMEOUT
|
||||
SQL_CONNECTION = cfg.CONF.sql_connection
|
||||
SQL_IDLE_TIMEOUT = cfg.CONF.sql_idle_timeout
|
||||
IMPL = db_api.DBAPI(backend_mapping=_BACKEND_MAPPING)
|
||||
|
||||
|
||||
def get_session():
|
||||
|
@ -14,13 +14,13 @@
|
||||
|
||||
"""Database setup and migration commands."""
|
||||
|
||||
from heat.db import utils
|
||||
from heat.openstack.common.db import api as db_api
|
||||
|
||||
|
||||
IMPL = utils.LazyPluggable('db_backend',
|
||||
sqlalchemy='heat.db.sqlalchemy.migration')
|
||||
|
||||
INIT_VERSION = 14
|
||||
_BACKEND_MAPPING = {'sqlalchemy': 'heat.db.sqlalchemy.migration'}
|
||||
|
||||
IMPL = db_api.DBAPI(backend_mapping=_BACKEND_MAPPING)
|
||||
|
||||
|
||||
def db_sync(version=None):
|
||||
|
@ -14,6 +14,7 @@
|
||||
# under the License.
|
||||
|
||||
'''Implementation of SQLAlchemy backend.'''
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
|
||||
@ -28,8 +29,17 @@ from heat.openstack.common.gettextutils import _
|
||||
from heat.common import crypt
|
||||
from heat.common import exception
|
||||
from heat.db.sqlalchemy import models
|
||||
from heat.db.sqlalchemy.session import get_engine
|
||||
from heat.db.sqlalchemy.session import get_session
|
||||
from heat.openstack.common.db.sqlalchemy import session as db_session
|
||||
|
||||
|
||||
get_engine = db_session.get_engine
|
||||
get_session = db_session.get_session
|
||||
|
||||
|
||||
def get_backend():
|
||||
"""The backend is this module itself."""
|
||||
|
||||
return sys.modules[__name__]
|
||||
|
||||
|
||||
def model_query(context, *args):
|
||||
@ -160,7 +170,7 @@ def resource_data_set(resource, key, value, redact=False):
|
||||
current.resource_id = resource.id
|
||||
current.redact = redact
|
||||
current.value = value
|
||||
current.save()
|
||||
current.save(session=resource.context.session)
|
||||
return current
|
||||
|
||||
|
||||
|
@ -16,7 +16,6 @@ import distutils.version as dist_version
|
||||
import os
|
||||
import sys
|
||||
|
||||
from heat.db.sqlalchemy.session import get_engine
|
||||
from heat.db import migration
|
||||
|
||||
import sqlalchemy
|
||||
@ -24,11 +23,18 @@ import migrate
|
||||
from migrate.versioning import util as migrate_util
|
||||
|
||||
from heat.openstack.common import exception
|
||||
from heat.openstack.common.db.sqlalchemy.session import get_engine
|
||||
from heat.openstack.common.gettextutils import _
|
||||
|
||||
_REPOSITORY = None
|
||||
|
||||
|
||||
def get_backend():
|
||||
"""The backend is this module itself."""
|
||||
|
||||
return sys.modules[__name__]
|
||||
|
||||
|
||||
@migrate_util.decorator
|
||||
def patched_with_engine(f, *a, **kw):
|
||||
url = a[0]
|
||||
|
@ -18,19 +18,19 @@ SQLAlchemy models for heat data.
|
||||
import sqlalchemy
|
||||
|
||||
from sqlalchemy.dialects import mysql
|
||||
from sqlalchemy.orm import relationship, backref, object_mapper
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import relationship, backref
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy import types
|
||||
from json import dumps
|
||||
from json import loads
|
||||
from heat.openstack.common import exception
|
||||
from heat.openstack.common import uuidutils
|
||||
from heat.openstack.common import timeutils
|
||||
from heat.db.sqlalchemy.session import get_session
|
||||
from heat.openstack.common.db.sqlalchemy import models
|
||||
from heat.openstack.common.db.sqlalchemy import session
|
||||
from sqlalchemy.orm.session import Session
|
||||
|
||||
BASE = declarative_base()
|
||||
get_session = session.get_session
|
||||
|
||||
|
||||
class Json(types.TypeDecorator):
|
||||
@ -58,29 +58,9 @@ except ImportError:
|
||||
MutableDict.associate_with(Json)
|
||||
|
||||
|
||||
class HeatBase(object):
|
||||
class HeatBase(models.ModelBase, models.TimestampMixin):
|
||||
"""Base class for Heat Models."""
|
||||
__table_args__ = {'mysql_engine': 'InnoDB'}
|
||||
__table_initialized__ = False
|
||||
created_at = sqlalchemy.Column(sqlalchemy.DateTime,
|
||||
default=timeutils.utcnow)
|
||||
updated_at = sqlalchemy.Column(sqlalchemy.DateTime,
|
||||
onupdate=timeutils.utcnow)
|
||||
|
||||
def save(self, session=None):
|
||||
"""Save this object."""
|
||||
if not session:
|
||||
session = Session.object_session(self)
|
||||
if not session:
|
||||
session = get_session()
|
||||
session.add(self)
|
||||
try:
|
||||
session.flush()
|
||||
except IntegrityError as e:
|
||||
if str(e).endswith('is not unique'):
|
||||
raise exception.Duplicate(str(e))
|
||||
else:
|
||||
raise
|
||||
|
||||
def expire(self, session=None, attrs=None):
|
||||
"""Expire this object ()."""
|
||||
@ -107,28 +87,6 @@ class HeatBase(object):
|
||||
session.delete(self)
|
||||
session.flush()
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
setattr(self, key, value)
|
||||
|
||||
def __getitem__(self, key):
|
||||
return getattr(self, key)
|
||||
|
||||
def get(self, key, default=None):
|
||||
return getattr(self, key, default)
|
||||
|
||||
def __iter__(self):
|
||||
self._i = iter(object_mapper(self).columns)
|
||||
return self
|
||||
|
||||
def next(self):
|
||||
n = self._i.next().name
|
||||
return n, getattr(self, n)
|
||||
|
||||
def update(self, values):
|
||||
"""Make the model object behave like a dict."""
|
||||
for k, v in values.iteritems():
|
||||
setattr(self, k, v)
|
||||
|
||||
def update_and_save(self, values, session=None):
|
||||
if not session:
|
||||
session = Session.object_session(self)
|
||||
@ -139,17 +97,6 @@ class HeatBase(object):
|
||||
setattr(self, k, v)
|
||||
session.commit()
|
||||
|
||||
def iteritems(self):
|
||||
"""Make the model object behave like a dict.
|
||||
|
||||
Includes attributes from joins.
|
||||
"""
|
||||
local = dict(self)
|
||||
joined = dict([(k, v) for k, v in self.__dict__.iteritems()
|
||||
if not k[0] == '_'])
|
||||
local.update(joined)
|
||||
return local.iteritems()
|
||||
|
||||
|
||||
class SoftDelete(object):
|
||||
deleted_at = sqlalchemy.Column(sqlalchemy.DateTime)
|
||||
|
@ -1,102 +0,0 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Session Handling for SQLAlchemy backend."""
|
||||
|
||||
import sqlalchemy.interfaces
|
||||
import sqlalchemy.orm
|
||||
import sqlalchemy.engine
|
||||
from sqlalchemy.exc import DisconnectionError
|
||||
|
||||
from heat.openstack.common import log as logging
|
||||
|
||||
from heat.db import api as db_api
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_ENGINE = None
|
||||
_MAKER = None
|
||||
|
||||
|
||||
def get_session(autocommit=True, expire_on_commit=False):
|
||||
"""Return a SQLAlchemy session."""
|
||||
global _MAKER
|
||||
|
||||
if _MAKER is None:
|
||||
_MAKER = get_maker(get_engine(), autocommit, expire_on_commit)
|
||||
return _MAKER()
|
||||
|
||||
|
||||
class SynchronousSwitchListener(sqlalchemy.interfaces.PoolListener):
|
||||
|
||||
"""Switch sqlite connections to non-synchronous mode."""
|
||||
|
||||
def connect(self, dbapi_con, con_record):
|
||||
dbapi_con.execute("PRAGMA synchronous = OFF")
|
||||
|
||||
|
||||
class MySQLPingListener(object):
|
||||
|
||||
"""
|
||||
Ensures that MySQL connections checked out of the
|
||||
pool are alive.
|
||||
|
||||
Borrowed from:
|
||||
http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f
|
||||
"""
|
||||
|
||||
def checkout(self, dbapi_con, con_record, con_proxy):
|
||||
try:
|
||||
dbapi_con.cursor().execute('select 1')
|
||||
except dbapi_con.OperationalError as ex:
|
||||
if ex.args[0] in (2006, 2013, 2014, 2045, 2055):
|
||||
logger.warn('Got mysql server has gone away: %s', ex)
|
||||
raise DisconnectionError("Database server went away")
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
def get_engine():
|
||||
"""Return a SQLAlchemy engine."""
|
||||
global _ENGINE
|
||||
if _ENGINE is None:
|
||||
connection_dict = sqlalchemy.engine.url.make_url(_get_sql_connection())
|
||||
engine_args = {
|
||||
"pool_recycle": _get_sql_idle_timeout(),
|
||||
"echo": False,
|
||||
'convert_unicode': True
|
||||
}
|
||||
|
||||
if 'mysql' in connection_dict.drivername:
|
||||
engine_args['listeners'] = [MySQLPingListener()]
|
||||
|
||||
_ENGINE = sqlalchemy.create_engine(_get_sql_connection(),
|
||||
**engine_args)
|
||||
return _ENGINE
|
||||
|
||||
|
||||
def get_maker(engine, autocommit=True, expire_on_commit=False):
|
||||
"""Return a SQLAlchemy sessionmaker using the given engine."""
|
||||
ses = sqlalchemy.orm.sessionmaker(
|
||||
bind=engine,
|
||||
autocommit=autocommit,
|
||||
expire_on_commit=expire_on_commit)
|
||||
return sqlalchemy.orm.scoped_session(ses)
|
||||
|
||||
|
||||
def _get_sql_connection():
|
||||
return db_api.SQL_CONNECTION
|
||||
|
||||
|
||||
def _get_sql_idle_timeout():
|
||||
return db_api.SQL_IDLE_TIMEOUT
|
@ -24,7 +24,6 @@ gettextutils.install('heat')
|
||||
|
||||
from oslo.config import cfg
|
||||
from heat.openstack.common import log as logging
|
||||
from heat.db import api
|
||||
from heat.db import migration
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
@ -36,8 +35,6 @@ if __name__ == '__main__':
|
||||
print('*******************************************', file=sys.stderr)
|
||||
cfg.CONF(project='heat', prog='heat-engine')
|
||||
|
||||
api.configure()
|
||||
|
||||
try:
|
||||
migration.db_sync()
|
||||
except Exception as exc:
|
||||
|
@ -133,8 +133,10 @@ class CloudWatchAlarmTest(HeatTestCase):
|
||||
scheduler.TaskRunner(rsrc.suspend)()
|
||||
self.assertEqual(rsrc.state, (rsrc.SUSPEND, rsrc.COMPLETE))
|
||||
|
||||
self.ctx = utils.dummy_context()
|
||||
|
||||
wr = watchrule.WatchRule.load(
|
||||
None, watch_name="test_stack-MEMAlarmHigh")
|
||||
self.ctx, watch_name="test_stack-MEMAlarmHigh")
|
||||
|
||||
self.assertEqual(wr.state, watchrule.WatchRule.SUSPENDED)
|
||||
|
||||
@ -142,7 +144,7 @@ class CloudWatchAlarmTest(HeatTestCase):
|
||||
self.assertEqual(rsrc.state, (rsrc.RESUME, rsrc.COMPLETE))
|
||||
|
||||
wr = watchrule.WatchRule.load(
|
||||
None, watch_name="test_stack-MEMAlarmHigh")
|
||||
self.ctx, watch_name="test_stack-MEMAlarmHigh")
|
||||
|
||||
self.assertEqual(wr.state, watchrule.WatchRule.NODATA)
|
||||
|
||||
|
@ -718,6 +718,9 @@ class StackServiceCreateUpdateDeleteTest(HeatTestCase):
|
||||
self.m.StubOutWithMock(parser.Stack, 'load')
|
||||
self.m.StubOutWithMock(parser, 'Template')
|
||||
self.m.StubOutWithMock(environment, 'Environment')
|
||||
self.m.StubOutWithMock(self.man, '_get_stack')
|
||||
|
||||
self.man._get_stack(self.ctx, old_stack.identifier()).AndReturn(s)
|
||||
|
||||
parser.Stack.load(self.ctx, stack=s).AndReturn(old_stack)
|
||||
|
||||
|
@ -137,7 +137,7 @@ class ResourceTest(HeatTestCase):
|
||||
self.assertEqual(res.status, res.IN_PROGRESS)
|
||||
self.assertEqual(res.status_reason, 'test_store')
|
||||
|
||||
db_res = r = db_api.resource_get(None, res.id)
|
||||
db_res = r = db_api.resource_get(res.context, res.id)
|
||||
self.assertEqual(db_res.action, res.CREATE)
|
||||
self.assertEqual(db_res.status, res.IN_PROGRESS)
|
||||
self.assertEqual(db_res.status_reason, 'test_store')
|
||||
|
@ -130,7 +130,7 @@ class SqlAlchemyTest(HeatTestCase):
|
||||
cs._store_or_update(cs.CREATE, cs.IN_PROGRESS, 'test_store')
|
||||
|
||||
cs.my_secret = 'fake secret'
|
||||
rs = db_api.resource_get_by_name_and_stack(None,
|
||||
rs = db_api.resource_get_by_name_and_stack(self.ctx,
|
||||
'cs_encryption',
|
||||
stack.id)
|
||||
encrypted_key = rs.data[0]['value']
|
||||
|
@ -20,13 +20,17 @@ import uuid
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from oslo.config import cfg
|
||||
|
||||
from heat.common import context
|
||||
from heat.common import exception
|
||||
from heat.engine import environment
|
||||
from heat.engine import parser
|
||||
|
||||
from heat.db.sqlalchemy.session import get_engine
|
||||
from heat.db import migration
|
||||
from heat.openstack.common.db.sqlalchemy import session
|
||||
|
||||
get_engine = session.get_engine
|
||||
|
||||
|
||||
class UUIDStub(object):
|
||||
@ -112,6 +116,8 @@ def wr_delete_after(test_fn):
|
||||
|
||||
|
||||
def setup_dummy_db():
|
||||
cfg.CONF.set_default('sqlite_synchronous', False)
|
||||
session.set_defaults(sql_connection="sqlite://", sqlite_db='heat.db')
|
||||
migration.db_sync()
|
||||
engine = get_engine()
|
||||
engine.connect()
|
||||
|
Loading…
Reference in New Issue
Block a user