Browse Source

cascade_service: DB infrastructure

Base DAL implementation and DevStack integration. Our database
schema is under design so the models may be changed later.

Partially implements: blueprint implement-dal

Change-Id: I8b16b3217e6b72e04bd8886d01d638f2d5a5c388
changes/46/209746/3
zhiyuan_cai 6 years ago
committed by Zhiyuan Cai
parent
commit
be30bb2fd8
  1. 36
      cmd/manage.py
  2. 3
      devstack/plugin.sh
  3. 63
      tricircle/context.py
  4. 0
      tricircle/db/__init__.py
  5. 139
      tricircle/db/core.py
  6. 24
      tricircle/db/exception.py
  7. 17
      tricircle/db/migrate_repo/__init__.py
  8. 26
      tricircle/db/migrate_repo/migrate.cfg
  9. 73
      tricircle/db/migrate_repo/versions/001_init.py
  10. 0
      tricircle/db/migrate_repo/versions/__init__.py
  11. 38
      tricircle/db/migration_helpers.py
  12. 97
      tricircle/db/models.py
  13. 0
      tricircle/tests/__init__.py
  14. 0
      tricircle/tests/unit/__init__.py
  15. 0
      tricircle/tests/unit/db/__init__.py
  16. 107
      tricircle/tests/unit/db/test_models.py

36
cmd/manage.py

@ -0,0 +1,36 @@
# Copyright 2015 Huawei Technologies Co., Ltd.
# All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sys
from oslo_config import cfg
from tricircle.db import core
import tricircle.db.migration_helpers as migration_helpers
def main(argv=None, config_files=None):
core.initialize()
cfg.CONF(args=argv[2:],
project='tricircle',
default_config_files=config_files)
migration_helpers.find_migrate_repo()
migration_helpers.sync_repo(1)
if __name__ == '__main__':
config_file = sys.argv[1]
main(argv=sys.argv, config_files=[config_file])

3
devstack/plugin.sh

@ -40,6 +40,9 @@ if [[ "$Q_ENABLE_TRICIRCLE" == "True" ]]; then
configure_tricircle_plugin
echo export PYTHONPATH=\$PYTHONPATH:$TRICIRCLE_DIR >> $RC_DIR/.localrc.auto
recreate_database tricircle
python "$TRICIRCLE_DIR/cmd/manage.py" "$TRICIRCLE_CASCADE_CONF"
elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
echo_summary "Initializing Cascading Service"

63
tricircle/context.py

@ -0,0 +1,63 @@
# Copyright 2015 Huawei Technologies Co., Ltd.
# All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_context import context as oslo_ctx
from tricircle.db import core
class ContextBase(oslo_ctx.RequestContext):
def __init__(self, auth_token=None, user_id=None, tenant_id=None,
is_admin=False, request_id=None, overwrite=True,
user_name=None, tenant_name=None, **kwargs):
super(ContextBase, self).__init__(
auth_token=auth_token,
user=user_id or kwargs.get('user', None),
tenant=tenant_id or kwargs.get('tenant', None),
domain=kwargs.get('domain', None),
user_domain=kwargs.get('user_domain', None),
project_domain=kwargs.get('project_domain', None),
is_admin=is_admin,
read_only=kwargs.get('read_only', False),
show_deleted=kwargs.get('show_deleted', False),
request_id=request_id,
resource_uuid=kwargs.get('resource_uuid', None),
overwrite=overwrite)
self.user_name = user_name
self.tenant_name = tenant_name
def to_dict(self):
ctx_dict = super(ContextBase, self).to_dict()
ctx_dict.update({
'user_name': self.user_name,
'tenant_name': self.tenant_name
})
return ctx_dict
@classmethod
def from_dict(cls, ctx):
return cls(**ctx)
class Context(ContextBase):
def __init__(self, **kwargs):
super(Context, self).__init__(**kwargs)
self._session = None
@property
def session(self):
if not self._session:
self._session = core.get_session()
return self._session

0
tricircle/db/__init__.py

139
tricircle/db/core.py

@ -0,0 +1,139 @@
# Copyright 2015 Huawei Technologies Co., Ltd.
# All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
import oslo_db.options as db_options
from oslo_db.sqlalchemy import session as db_session
from oslo_utils import strutils
import sqlalchemy as sql
from sqlalchemy.ext import declarative
from sqlalchemy.inspection import inspect
import tricircle.db.exception as db_exception
_engine_facade = None
ModelBase = declarative.declarative_base()
def _filter_query(model, query, filters):
"""Apply filter to query
:param model:
:param query:
:param filters: list of filter dict with key 'key', 'comparator', 'value'
like {'key': 'site_id', 'comparator': 'eq', 'value': 'test_site_uuid'}
:return:
"""
filter_dict = {}
for query_filter in filters:
# only eq filter supported at first
if query_filter['comparator'] != 'eq':
continue
key = query_filter['key']
if key not in model.attributes:
continue
if isinstance(inspect(model).columns[key].type, sql.Boolean):
filter_dict[key] = strutils.bool_from_string(query_filter['value'])
else:
filter_dict[key] = query_filter['value']
if filter_dict:
return query.filter_by(**filter_dict)
else:
return query
def _get_engine_facade():
global _engine_facade
if not _engine_facade:
_engine_facade = db_session.EngineFacade.from_config(cfg.CONF)
return _engine_facade
def _get_resource(context, model, pk_value):
res_obj = context.session.query(model).get(pk_value)
if not res_obj:
raise db_exception.ResourceNotFound(model, pk_value)
return res_obj
def create_resource(context, model, res_dict):
res_obj = model.from_dict(res_dict)
context.session.add(res_obj)
return res_obj.to_dict()
def delete_resource(context, model, pk_value):
res_obj = _get_resource(context, model, pk_value)
context.session.delete(res_obj)
def get_engine():
return _get_engine_facade().get_engine()
def get_resource(context, model, pk_value):
return _get_resource(context, model, pk_value).to_dict()
def get_session(expire_on_commit=False):
return _get_engine_facade().get_session(expire_on_commit=expire_on_commit)
def initialize():
db_options.set_defaults(
cfg.CONF,
connection='sqlite:///:memory:')
def query_resource(context, model, filters):
query = context.session.query(model)
objs = _filter_query(model, query, filters)
return [obj.to_dict() for obj in objs]
def update_resource(context, model, pk_value, update_dict):
res_obj = _get_resource(context, model, pk_value)
for key in update_dict:
if key not in model.attributes:
continue
skip = False
for pkey in inspect(model).primary_key:
if pkey.name == key:
skip = True
break
if skip:
continue
setattr(res_obj, key, update_dict[key])
return res_obj.to_dict()
class DictBase(object):
attributes = []
@classmethod
def from_dict(cls, d):
return cls(**d)
def to_dict(self):
d = {}
for attr in self.__class__.attributes:
d[attr] = getattr(self, attr)
return d
def __getitem__(self, key):
return getattr(self, key)

24
tricircle/db/exception.py

@ -0,0 +1,24 @@
# Copyright 2015 Huawei Technologies Co., Ltd.
# All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
class ResourceNotFound(Exception):
def __init__(self, model, pk_value):
res_type = model.__name__.lower()
message = "Could not find %(res_type)s: %(pk_value)s" % {
'res_type': res_type,
'pk_value': pk_value
}
super(ResourceNotFound, self).__init__(message)

17
tricircle/db/migrate_repo/__init__.py

@ -0,0 +1,17 @@
# Copyright 2015 Huawei Technologies Co., Ltd.
# All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
DB_INIT_VERSION = 0

26
tricircle/db/migrate_repo/migrate.cfg

@ -0,0 +1,26 @@
[db_settings]
# Used to identify which repository this database is versioned under.
# You can use the name of your project.
repository_id=tricircle
# The name of the database table used to track the schema version.
# This name shouldn't already be used by your project.
# If this is changed once a database is under version control, you'll need to
# change the table name in each database too.
version_table=migrate_version
# When committing a change script, Migrate will attempt to generate the
# sql for all supported databases; normally, if one of them fails - probably
# because you don't have that database installed - it is ignored and the
# commit continues, perhaps ending successfully.
# Databases in this list MUST compile successfully during a commit, or the
# entire commit will fail. List the databases your application will actually
# be using to ensure your updates to that database work properly.
# This must be a list; example: ['postgres','sqlite']
required_dbs=[]
# When creating new change scripts, Migrate will stamp the new script with
# a version number. By default this is latest_version + 1. You can set this
# to 'true' to tell Migrate to use the UTC timestamp instead.
use_timestamp_numbering=False

73
tricircle/db/migrate_repo/versions/001_init.py

@ -0,0 +1,73 @@
# Copyright 2015 Huawei Technologies Co., Ltd.
# All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import migrate
import sqlalchemy as sql
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
cascaded_sites = sql.Table(
'cascaded_sites', meta,
sql.Column('site_id', sql.String(length=64), primary_key=True),
sql.Column('site_name', sql.String(length=64), unique=True,
nullable=False),
sql.Column('az_id', sql.String(length=64), nullable=False),
mysql_engine='InnoDB',
mysql_charset='utf8')
cascaded_site_service_configuration = sql.Table(
'cascaded_site_service_configuration', meta,
sql.Column('service_id', sql.String(length=64), primary_key=True),
sql.Column('site_id', sql.String(length=64), nullable=False),
sql.Column('service_name', sql.String(length=64), unique=True,
nullable=False),
sql.Column('service_type', sql.String(length=64), nullable=False),
sql.Column('service_url', sql.String(length=512), nullable=False),
mysql_engine='InnoDB',
mysql_charset='utf8')
cascaded_service_types = sql.Table(
'cascaded_service_types', meta,
sql.Column('id', sql.Integer, primary_key=True),
sql.Column('service_type', sql.String(length=64), unique=True),
mysql_engine='InnoDB',
mysql_charset='utf8')
cascaded_site_services = sql.Table(
'cascaded_site_services', meta,
sql.Column('site_id', sql.String(length=64), primary_key=True),
mysql_engine='InnoDB',
mysql_charset='utf8')
tables = [cascaded_sites, cascaded_site_service_configuration,
cascaded_service_types, cascaded_site_services]
for table in tables:
table.create()
fkeys = [
{'columns': [cascaded_site_service_configuration.c.site_id],
'references': [cascaded_sites.c.site_id]},
{'columns': [cascaded_site_service_configuration.c.service_type],
'references': [cascaded_service_types.c.service_type]}
]
for fkey in fkeys:
migrate.ForeignKeyConstraint(columns=fkey['columns'],
refcolumns=fkey['references'],
name=fkey.get('name')).create()
def downgrade(migrate_engine):
raise NotImplementedError('can not downgrade from init repo.')

0
tricircle/db/migrate_repo/versions/__init__.py

38
tricircle/db/migration_helpers.py

@ -0,0 +1,38 @@
# Copyright 2015 Huawei Technologies Co., Ltd.
# All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from oslo_db.sqlalchemy import migration
from tricircle import db
from tricircle.db import core
from tricircle.db import migrate_repo
def find_migrate_repo(package=None, repo_name='migrate_repo'):
package = package or db
path = os.path.abspath(os.path.join(
os.path.dirname(package.__file__), repo_name))
# TODO(zhiyuan) handle path not valid exception
return path
def sync_repo(version):
repo_abs_path = find_migrate_repo()
init_version = migrate_repo.DB_INIT_VERSION
engine = core.get_engine()
migration.db_sync(engine, repo_abs_path, version, init_version)

97
tricircle/db/models.py

@ -0,0 +1,97 @@
# Copyright 2015 Huawei Technologies Co., Ltd.
# All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sql
from tricircle.db import core
def create_site(context, site_dict):
with context.session.begin():
return core.create_resource(context, Site, site_dict)
def delete_site(context, site_id):
with context.session.begin():
return core.delete_resource(context, Site, site_id)
def get_site(context, site_id):
with context.session.begin():
return core.get_resource(context, Site, site_id)
def list_sites(context, filters):
with context.session.begin():
return core.query_resource(context, Site, filters)
def update_site(context, site_id, update_dict):
with context.session.begin():
return core.update_resource(context, Site, site_id, update_dict)
def create_service_type(context, type_dict):
with context.session.begin():
return core.create_resource(context, ServiceType, type_dict)
def create_site_service_configuration(context, config_dict):
with context.session.begin():
return core.create_resource(context, SiteServiceConfiguration,
config_dict)
class Site(core.ModelBase, core.DictBase):
__tablename__ = 'cascaded_sites'
attributes = ['site_id', 'site_name', 'az_id']
site_id = sql.Column('site_id', sql.String(length=64), primary_key=True)
site_name = sql.Column('site_name', sql.String(length=64), unique=True,
nullable=False)
az_id = sql.Column('az_id', sql.String(length=64), nullable=False)
class SiteServiceConfiguration(core.ModelBase, core.DictBase):
__tablename__ = 'cascaded_site_service_configuration'
attributes = ['service_id', 'site_id', 'service_name',
'service_type', 'service_url']
service_id = sql.Column('service_id', sql.String(length=64),
primary_key=True)
site_id = sql.Column('site_id', sql.String(length=64),
sql.ForeignKey('cascaded_sites.site_id'),
nullable=False)
service_name = sql.Column('service_name', sql.String(length=64),
unique=True, nullable=False)
service_type = sql.Column(
'service_type', sql.String(length=64),
sql.ForeignKey('cascaded_service_types.service_type'),
nullable=False)
service_url = sql.Column('service_url', sql.String(length=512),
nullable=False)
class ServiceType(core.ModelBase, core.DictBase):
__tablename__ = 'cascaded_service_types'
attributes = ['id', 'service_type']
id = sql.Column('id', sql.Integer, primary_key=True)
service_type = sql.Column('service_type', sql.String(length=64),
unique=True)
class SiteService(core.ModelBase, core.DictBase):
__tablename__ = 'cascaded_site_services'
attributes = ['site_id']
site_id = sql.Column('site_id', sql.String(length=64), primary_key=True)

0
tricircle/tests/__init__.py

0
tricircle/tests/unit/__init__.py

0
tricircle/tests/unit/db/__init__.py

107
tricircle/tests/unit/db/test_models.py

@ -0,0 +1,107 @@
# Copyright 2015 Huawei Technologies Co., Ltd.
# All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
from tricircle import context
from tricircle.db import core
from tricircle.db import exception
from tricircle.db import models
class ModelsTest(unittest.TestCase):
def setUp(self):
core.initialize()
core.ModelBase.metadata.create_all(core.get_engine())
self.context = context.Context()
def test_obj_to_dict(self):
site = {'site_id': 'test_site_uuid',
'site_name': 'test_site',
'az_id': 'test_az_uuid'}
site_obj = models.Site.from_dict(site)
for attr in site_obj.attributes:
self.assertEqual(getattr(site_obj, attr), site[attr])
def test_create(self):
site = {'site_id': 'test_site_uuid',
'site_name': 'test_site',
'az_id': 'test_az_uuid'}
site_ret = models.create_site(self.context, site)
self.assertEqual(site_ret, site)
service_type = {'id': 1,
'service_type': 'nova'}
type_ret = models.create_service_type(self.context, service_type)
self.assertEqual(type_ret, service_type)
configuration = {
'service_id': 'test_config_uuid',
'site_id': 'test_site_uuid',
'service_name': 'nova_service',
'service_type': 'nova',
'service_url': 'http://test_url'
}
config_ret = models.create_site_service_configuration(self.context,
configuration)
self.assertEqual(config_ret, configuration)
def test_update(self):
site = {'site_id': 'test_site_uuid',
'site_name': 'test_site',
'az_id': 'test_az1_uuid'}
models.create_site(self.context, site)
update_dict = {'site_id': 'fake_uuid',
'site_name': 'test_site2',
'az_id': 'test_az2_uuid'}
ret = models.update_site(self.context, 'test_site_uuid', update_dict)
# primary key value will not be updated
self.assertEqual(ret['site_id'], 'test_site_uuid')
self.assertEqual(ret['site_name'], 'test_site2')
self.assertEqual(ret['az_id'], 'test_az2_uuid')
def test_delete(self):
site = {'site_id': 'test_site_uuid',
'site_name': 'test_site',
'az_id': 'test_az_uuid'}
models.create_site(self.context, site)
models.delete_site(self.context, 'test_site_uuid')
self.assertRaises(exception.ResourceNotFound, models.get_site,
self.context, 'test_site_uuid')
def test_query(self):
site1 = {'site_id': 'test_site1_uuid',
'site_name': 'test_site1',
'az_id': 'test_az1_uuid'}
site2 = {'site_id': 'test_site2_uuid',
'site_name': 'test_site2',
'az_id': 'test_az2_uuid'}
models.create_site(self.context, site1)
models.create_site(self.context, site2)
filters = [{'key': 'site_name',
'comparator': 'eq',
'value': 'test_site2'}]
sites = models.list_sites(self.context, filters)
self.assertEqual(len(sites), 1)
self.assertEqual(sites[0], site2)
filters = [{'key': 'site_name',
'comparator': 'eq',
'value': 'test_site3'}]
sites = models.list_sites(self.context, filters)
self.assertEqual(len(sites), 0)
def tearDown(self):
core.ModelBase.metadata.drop_all(core.get_engine())
Loading…
Cancel
Save