cascade_service: DB infrastructure

Base DAL implementation and DevStack integration. Our database
schema is under design so the models may be changed later.

Partially implements: blueprint implement-dal

Change-Id: I8b16b3217e6b72e04bd8886d01d638f2d5a5c388

cmd/manage.py

@@ -0,0 +1,36 @@
+# Copyright 2015 Huawei Technologies Co., Ltd.
+# All Rights Reserved
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import sys
+
+from oslo_config import cfg
+
+from tricircle.db import core
+import tricircle.db.migration_helpers as migration_helpers
+
+
+def main(argv=None, config_files=None):
+    core.initialize()
+    cfg.CONF(args=argv[2:],
+             project='tricircle',
+             default_config_files=config_files)
+    migration_helpers.find_migrate_repo()
+    migration_helpers.sync_repo(1)
+
+
+if __name__ == '__main__':
+    config_file = sys.argv[1]
+    main(argv=sys.argv, config_files=[config_file])

devstack/plugin.sh

@@ -40,6 +40,9 @@ if [[ "$Q_ENABLE_TRICIRCLE" == "True" ]]; then
         configure_tricircle_plugin
         echo export PYTHONPATH=\$PYTHONPATH:$TRICIRCLE_DIR >> $RC_DIR/.localrc.auto
 
+        recreate_database tricircle
+        python "$TRICIRCLE_DIR/cmd/manage.py" "$TRICIRCLE_CASCADE_CONF"
+
     elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
         echo_summary "Initializing Cascading Service"
 

tricircle/context.py

@@ -0,0 +1,63 @@
+# Copyright 2015 Huawei Technologies Co., Ltd.
+# All Rights Reserved
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_context import context as oslo_ctx
+
+from tricircle.db import core
+
+
+class ContextBase(oslo_ctx.RequestContext):
+    def __init__(self, auth_token=None, user_id=None, tenant_id=None,
+                 is_admin=False, request_id=None, overwrite=True,
+                 user_name=None, tenant_name=None, **kwargs):
+        super(ContextBase, self).__init__(
+            auth_token=auth_token,
+            user=user_id or kwargs.get('user', None),
+            tenant=tenant_id or kwargs.get('tenant', None),
+            domain=kwargs.get('domain', None),
+            user_domain=kwargs.get('user_domain', None),
+            project_domain=kwargs.get('project_domain', None),
+            is_admin=is_admin,
+            read_only=kwargs.get('read_only', False),
+            show_deleted=kwargs.get('show_deleted', False),
+            request_id=request_id,
+            resource_uuid=kwargs.get('resource_uuid', None),
+            overwrite=overwrite)
+        self.user_name = user_name
+        self.tenant_name = tenant_name
+
+    def to_dict(self):
+        ctx_dict = super(ContextBase, self).to_dict()
+        ctx_dict.update({
+            'user_name': self.user_name,
+            'tenant_name': self.tenant_name
+        })
+        return ctx_dict
+
+    @classmethod
+    def from_dict(cls, ctx):
+        return cls(**ctx)
+
+
+class Context(ContextBase):
+    def __init__(self, **kwargs):
+        super(Context, self).__init__(**kwargs)
+        self._session = None
+
+    @property
+    def session(self):
+        if not self._session:
+            self._session = core.get_session()
+        return self._session

tricircle/db/core.py

@@ -0,0 +1,139 @@
+# Copyright 2015 Huawei Technologies Co., Ltd.
+# All Rights Reserved
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from oslo_config import cfg
+import oslo_db.options as db_options
+from oslo_db.sqlalchemy import session as db_session
+from oslo_utils import strutils
+import sqlalchemy as sql
+from sqlalchemy.ext import declarative
+from sqlalchemy.inspection import inspect
+
+import tricircle.db.exception as db_exception
+
+_engine_facade = None
+ModelBase = declarative.declarative_base()
+
+
+def _filter_query(model, query, filters):
+    """Apply filter to query
+    :param model:
+    :param query:
+    :param filters: list of filter dict with key 'key', 'comparator', 'value'
+    like {'key': 'site_id', 'comparator': 'eq', 'value': 'test_site_uuid'}
+    :return:
+    """
+    filter_dict = {}
+    for query_filter in filters:
+        # only eq filter supported at first
+        if query_filter['comparator'] != 'eq':
+            continue
+
+        key = query_filter['key']
+        if key not in model.attributes:
+            continue
+        if isinstance(inspect(model).columns[key].type, sql.Boolean):
+            filter_dict[key] = strutils.bool_from_string(query_filter['value'])
+        else:
+            filter_dict[key] = query_filter['value']
+    if filter_dict:
+        return query.filter_by(**filter_dict)
+    else:
+        return query
+
+
+def _get_engine_facade():
+    global _engine_facade
+
+    if not _engine_facade:
+        _engine_facade = db_session.EngineFacade.from_config(cfg.CONF)
+
+    return _engine_facade
+
+
+def _get_resource(context, model, pk_value):
+    res_obj = context.session.query(model).get(pk_value)
+    if not res_obj:
+        raise db_exception.ResourceNotFound(model, pk_value)
+    return res_obj
+
+
+def create_resource(context, model, res_dict):
+    res_obj = model.from_dict(res_dict)
+    context.session.add(res_obj)
+    return res_obj.to_dict()
+
+
+def delete_resource(context, model, pk_value):
+    res_obj = _get_resource(context, model, pk_value)
+    context.session.delete(res_obj)
+
+
+def get_engine():
+    return _get_engine_facade().get_engine()
+
+
+def get_resource(context, model, pk_value):
+    return _get_resource(context, model, pk_value).to_dict()
+
+
+def get_session(expire_on_commit=False):
+    return _get_engine_facade().get_session(expire_on_commit=expire_on_commit)
+
+
+def initialize():
+    db_options.set_defaults(
+        cfg.CONF,
+        connection='sqlite:///:memory:')
+
+
+def query_resource(context, model, filters):
+    query = context.session.query(model)
+    objs = _filter_query(model, query, filters)
+    return [obj.to_dict() for obj in objs]
+
+
+def update_resource(context, model, pk_value, update_dict):
+    res_obj = _get_resource(context, model, pk_value)
+    for key in update_dict:
+        if key not in model.attributes:
+            continue
+        skip = False
+        for pkey in inspect(model).primary_key:
+            if pkey.name == key:
+                skip = True
+                break
+        if skip:
+            continue
+        setattr(res_obj, key, update_dict[key])
+    return res_obj.to_dict()
+
+
+class DictBase(object):
+    attributes = []
+
+    @classmethod
+    def from_dict(cls, d):
+        return cls(**d)
+
+    def to_dict(self):
+        d = {}
+        for attr in self.__class__.attributes:
+            d[attr] = getattr(self, attr)
+        return d
+
+    def __getitem__(self, key):
+        return getattr(self, key)

tricircle/db/exception.py

@@ -0,0 +1,24 @@
+# Copyright 2015 Huawei Technologies Co., Ltd.
+# All Rights Reserved
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+class ResourceNotFound(Exception):
+    def __init__(self, model, pk_value):
+        res_type = model.__name__.lower()
+        message = "Could not find %(res_type)s: %(pk_value)s" % {
+            'res_type': res_type,
+            'pk_value': pk_value
+        }
+        super(ResourceNotFound, self).__init__(message)

tricircle/db/migrate_repo/__init__.py

@@ -0,0 +1,17 @@
+# Copyright 2015 Huawei Technologies Co., Ltd.
+# All Rights Reserved
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+DB_INIT_VERSION = 0

tricircle/db/migrate_repo/migrate.cfg

@@ -0,0 +1,26 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=tricircle
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfully during a commit, or the
+# entire commit will fail. List the databases your application will actually
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]
+
+# When creating new change scripts, Migrate will stamp the new script with
+# a version number. By default this is latest_version + 1. You can set this
+# to 'true' to tell Migrate to use the UTC timestamp instead.
+use_timestamp_numbering=False
+

tricircle/db/migrate_repo/versions/001_init.py

@@ -0,0 +1,73 @@
+# Copyright 2015 Huawei Technologies Co., Ltd.
+# All Rights Reserved
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import migrate
+import sqlalchemy as sql
+
+
+def upgrade(migrate_engine):
+    meta = sql.MetaData()
+    meta.bind = migrate_engine
+
+    cascaded_sites = sql.Table(
+        'cascaded_sites', meta,
+        sql.Column('site_id', sql.String(length=64), primary_key=True),
+        sql.Column('site_name', sql.String(length=64), unique=True,
+                   nullable=False),
+        sql.Column('az_id', sql.String(length=64), nullable=False),
+        mysql_engine='InnoDB',
+        mysql_charset='utf8')
+    cascaded_site_service_configuration = sql.Table(
+        'cascaded_site_service_configuration', meta,
+        sql.Column('service_id', sql.String(length=64), primary_key=True),
+        sql.Column('site_id', sql.String(length=64), nullable=False),
+        sql.Column('service_name', sql.String(length=64), unique=True,
+                   nullable=False),
+        sql.Column('service_type', sql.String(length=64), nullable=False),
+        sql.Column('service_url', sql.String(length=512), nullable=False),
+        mysql_engine='InnoDB',
+        mysql_charset='utf8')
+    cascaded_service_types = sql.Table(
+        'cascaded_service_types', meta,
+        sql.Column('id', sql.Integer, primary_key=True),
+        sql.Column('service_type', sql.String(length=64), unique=True),
+        mysql_engine='InnoDB',
+        mysql_charset='utf8')
+    cascaded_site_services = sql.Table(
+        'cascaded_site_services', meta,
+        sql.Column('site_id', sql.String(length=64), primary_key=True),
+        mysql_engine='InnoDB',
+        mysql_charset='utf8')
+
+    tables = [cascaded_sites, cascaded_site_service_configuration,
+              cascaded_service_types, cascaded_site_services]
+    for table in tables:
+        table.create()
+
+    fkeys = [
+        {'columns': [cascaded_site_service_configuration.c.site_id],
+         'references': [cascaded_sites.c.site_id]},
+        {'columns': [cascaded_site_service_configuration.c.service_type],
+         'references': [cascaded_service_types.c.service_type]}
+    ]
+    for fkey in fkeys:
+        migrate.ForeignKeyConstraint(columns=fkey['columns'],
+                                     refcolumns=fkey['references'],
+                                     name=fkey.get('name')).create()
+
+
+def downgrade(migrate_engine):
+    raise NotImplementedError('can not downgrade from init repo.')

tricircle/db/migration_helpers.py

@@ -0,0 +1,38 @@
+# Copyright 2015 Huawei Technologies Co., Ltd.
+# All Rights Reserved
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import os
+
+from oslo_db.sqlalchemy import migration
+
+from tricircle import db
+from tricircle.db import core
+from tricircle.db import migrate_repo
+
+
+def find_migrate_repo(package=None, repo_name='migrate_repo'):
+    package = package or db
+    path = os.path.abspath(os.path.join(
+        os.path.dirname(package.__file__), repo_name))
+    # TODO(zhiyuan) handle path not valid exception
+    return path
+
+
+def sync_repo(version):
+    repo_abs_path = find_migrate_repo()
+    init_version = migrate_repo.DB_INIT_VERSION
+    engine = core.get_engine()
+    migration.db_sync(engine, repo_abs_path, version, init_version)

tricircle/db/models.py

@@ -0,0 +1,97 @@
+# Copyright 2015 Huawei Technologies Co., Ltd.
+# All Rights Reserved
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import sqlalchemy as sql
+
+from tricircle.db import core
+
+
+def create_site(context, site_dict):
+    with context.session.begin():
+        return core.create_resource(context, Site, site_dict)
+
+
+def delete_site(context, site_id):
+    with context.session.begin():
+        return core.delete_resource(context, Site, site_id)
+
+
+def get_site(context, site_id):
+    with context.session.begin():
+        return core.get_resource(context, Site, site_id)
+
+
+def list_sites(context, filters):
+    with context.session.begin():
+        return core.query_resource(context, Site, filters)
+
+
+def update_site(context, site_id, update_dict):
+    with context.session.begin():
+        return core.update_resource(context, Site, site_id, update_dict)
+
+
+def create_service_type(context, type_dict):
+    with context.session.begin():
+        return core.create_resource(context, ServiceType, type_dict)
+
+
+def create_site_service_configuration(context, config_dict):
+    with context.session.begin():
+        return core.create_resource(context, SiteServiceConfiguration,
+                                    config_dict)
+
+
+class Site(core.ModelBase, core.DictBase):
+    __tablename__ = 'cascaded_sites'
+    attributes = ['site_id', 'site_name', 'az_id']
+    site_id = sql.Column('site_id', sql.String(length=64), primary_key=True)
+    site_name = sql.Column('site_name', sql.String(length=64), unique=True,
+                           nullable=False)
+    az_id = sql.Column('az_id', sql.String(length=64), nullable=False)
+
+
+class SiteServiceConfiguration(core.ModelBase, core.DictBase):
+    __tablename__ = 'cascaded_site_service_configuration'
+    attributes = ['service_id', 'site_id', 'service_name',
+                  'service_type', 'service_url']
+    service_id = sql.Column('service_id', sql.String(length=64),
+                            primary_key=True)
+    site_id = sql.Column('site_id', sql.String(length=64),
+                         sql.ForeignKey('cascaded_sites.site_id'),
+                         nullable=False)
+    service_name = sql.Column('service_name', sql.String(length=64),
+                              unique=True, nullable=False)
+    service_type = sql.Column(
+        'service_type', sql.String(length=64),
+        sql.ForeignKey('cascaded_service_types.service_type'),
+        nullable=False)
+    service_url = sql.Column('service_url', sql.String(length=512),
+                             nullable=False)
+
+
+class ServiceType(core.ModelBase, core.DictBase):
+    __tablename__ = 'cascaded_service_types'
+    attributes = ['id', 'service_type']
+    id = sql.Column('id', sql.Integer, primary_key=True)
+    service_type = sql.Column('service_type', sql.String(length=64),
+                              unique=True)
+
+
+class SiteService(core.ModelBase, core.DictBase):
+    __tablename__ = 'cascaded_site_services'
+    attributes = ['site_id']
+    site_id = sql.Column('site_id', sql.String(length=64), primary_key=True)

tricircle/tests/unit/db/test_models.py

@@ -0,0 +1,107 @@
+# Copyright 2015 Huawei Technologies Co., Ltd.
+# All Rights Reserved
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import unittest
+
+from tricircle import context
+from tricircle.db import core
+from tricircle.db import exception
+from tricircle.db import models
+
+
+class ModelsTest(unittest.TestCase):
+    def setUp(self):
+        core.initialize()
+        core.ModelBase.metadata.create_all(core.get_engine())
+        self.context = context.Context()
+
+    def test_obj_to_dict(self):
+        site = {'site_id': 'test_site_uuid',
+                'site_name': 'test_site',
+                'az_id': 'test_az_uuid'}
+        site_obj = models.Site.from_dict(site)
+        for attr in site_obj.attributes:
+            self.assertEqual(getattr(site_obj, attr), site[attr])
+
+    def test_create(self):
+        site = {'site_id': 'test_site_uuid',
+                'site_name': 'test_site',
+                'az_id': 'test_az_uuid'}
+        site_ret = models.create_site(self.context, site)
+        self.assertEqual(site_ret, site)
+
+        service_type = {'id': 1,
+                        'service_type': 'nova'}
+        type_ret = models.create_service_type(self.context, service_type)
+        self.assertEqual(type_ret, service_type)
+
+        configuration = {
+            'service_id': 'test_config_uuid',
+            'site_id': 'test_site_uuid',
+            'service_name': 'nova_service',
+            'service_type': 'nova',
+            'service_url': 'http://test_url'
+        }
+        config_ret = models.create_site_service_configuration(self.context,
+                                                              configuration)
+        self.assertEqual(config_ret, configuration)
+
+    def test_update(self):
+        site = {'site_id': 'test_site_uuid',
+                'site_name': 'test_site',
+                'az_id': 'test_az1_uuid'}
+        models.create_site(self.context, site)
+        update_dict = {'site_id': 'fake_uuid',
+                       'site_name': 'test_site2',
+                       'az_id': 'test_az2_uuid'}
+        ret = models.update_site(self.context, 'test_site_uuid', update_dict)
+        # primary key value will not be updated
+        self.assertEqual(ret['site_id'], 'test_site_uuid')
+        self.assertEqual(ret['site_name'], 'test_site2')
+        self.assertEqual(ret['az_id'], 'test_az2_uuid')
+
+    def test_delete(self):
+        site = {'site_id': 'test_site_uuid',
+                'site_name': 'test_site',
+                'az_id': 'test_az_uuid'}
+        models.create_site(self.context, site)
+        models.delete_site(self.context, 'test_site_uuid')
+        self.assertRaises(exception.ResourceNotFound, models.get_site,
+                          self.context, 'test_site_uuid')
+
+    def test_query(self):
+        site1 = {'site_id': 'test_site1_uuid',
+                 'site_name': 'test_site1',
+                 'az_id': 'test_az1_uuid'}
+        site2 = {'site_id': 'test_site2_uuid',
+                 'site_name': 'test_site2',
+                 'az_id': 'test_az2_uuid'}
+        models.create_site(self.context, site1)
+        models.create_site(self.context, site2)
+        filters = [{'key': 'site_name',
+                    'comparator': 'eq',
+                    'value': 'test_site2'}]
+        sites = models.list_sites(self.context, filters)
+        self.assertEqual(len(sites), 1)
+        self.assertEqual(sites[0], site2)
+        filters = [{'key': 'site_name',
+                    'comparator': 'eq',
+                    'value': 'test_site3'}]
+        sites = models.list_sites(self.context, filters)
+        self.assertEqual(len(sites), 0)
+
+    def tearDown(self):
+        core.ModelBase.metadata.drop_all(core.get_engine())