Add db module

Change-Id: Iaf75cda465a63d52a9b5e22b8c3fe235a84800ed

nimble/cmd/dbsync.py

@@ -0,0 +1,91 @@
+# Copyright 2016 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Run storage database migration.
+"""
+
+import sys
+
+from oslo_config import cfg
+
+from nimble.common.i18n import _
+from nimble.common import service
+from nimble.conf import CONF
+from nimble.db import migration
+
+
+class DBCommand(object):
+
+    def upgrade(self):
+        migration.upgrade(CONF.command.revision)
+
+    def revision(self):
+        migration.revision(CONF.command.message, CONF.command.autogenerate)
+
+    def stamp(self):
+        migration.stamp(CONF.command.revision)
+
+    def version(self):
+        print(migration.version())
+
+    def create_schema(self):
+        migration.create_schema()
+
+
+def add_command_parsers(subparsers):
+    command_object = DBCommand()
+
+    parser = subparsers.add_parser(
+        'upgrade',
+        help=_("Upgrade the database schema to the latest version. "
+               "Optionally, use --revision to specify an alembic revision "
+               "string to upgrade to."))
+    parser.set_defaults(func=command_object.upgrade)
+    parser.add_argument('--revision', nargs='?')
+
+    parser = subparsers.add_parser('stamp')
+    parser.add_argument('--revision', nargs='?')
+    parser.set_defaults(func=command_object.stamp)
+
+    parser = subparsers.add_parser(
+        'revision',
+        help=_("Create a new alembic revision. "
+               "Use --message to set the message string."))
+    parser.add_argument('-m', '--message')
+    parser.add_argument('--autogenerate', action='store_true')
+    parser.set_defaults(func=command_object.revision)
+
+    parser = subparsers.add_parser(
+        'version',
+        help=_("Print the current version information and exit."))
+    parser.set_defaults(func=command_object.version)
+
+    parser = subparsers.add_parser(
+        'create_schema',
+        help=_("Create the database schema."))
+    parser.set_defaults(func=command_object.create_schema)
+
+
+def main():
+    command_opt = cfg.SubCommandOpt('command',
+                                    title='Command',
+                                    help=_('Available commands'),
+                                    handler=add_command_parsers)
+
+    CONF.register_cli_opt(command_opt)
+
+    service.prepare_service(sys.argv)
+    CONF.command.func()

nimble/common/paths.py

@@ -0,0 +1,50 @@
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+# Copyright 2012 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+from nimble.conf import CONF
+
+
+def basedir_def(*args):
+    """Return an uninterpolated path relative to $pybasedir."""
+    return os.path.join('$pybasedir', *args)
+
+
+def bindir_def(*args):
+    """Return an uninterpolated path relative to $bindir."""
+    return os.path.join('$bindir', *args)
+
+
+def state_path_def(*args):
+    """Return an uninterpolated path relative to $state_path."""
+    return os.path.join('$state_path', *args)
+
+
+def basedir_rel(*args):
+    """Return a path relative to $pybasedir."""
+    return os.path.join(CONF.pybasedir, *args)
+
+
+def bindir_rel(*args):
+    """Return a path relative to $bindir."""
+    return os.path.join(CONF.bindir, *args)
+
+
+def state_path_rel(*args):
+    """Return a path relative to $state_path."""
+    return os.path.join(CONF.state_path, *args)

nimble/conf/__init__.py

@@ -16,7 +16,9 @@
 from oslo_config import cfg
 
 from nimble.conf import api
+from nimble.conf import database
 
 CONF = cfg.CONF
 
 api.register_opts(CONF)
+database.register_opts(CONF)

nimble/conf/database.py

@@ -0,0 +1,28 @@
+# Copyright 2016 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+
+from nimble.common.i18n import _
+
+opts = [
+    cfg.StrOpt('mysql_engine',
+               default='InnoDB',
+               help=_('MySQL engine to use.'))
+]
+
+
+def register_opts(conf):
+    conf.register_opts(opts, group='database')

nimble/db/api.py

@@ -0,0 +1,42 @@
+# Copyright 2016 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""
+Base classes for storage engines
+"""
+
+import abc
+
+from oslo_config import cfg
+from oslo_db import api as db_api
+import six
+
+
+_BACKEND_MAPPING = {'sqlalchemy': 'nimble.db.sqlalchemy.api'}
+IMPL = db_api.DBAPI.from_config(cfg.CONF, backend_mapping=_BACKEND_MAPPING,
+                                lazy=True)
+
+
+def get_instance():
+    """Return a DB API instance."""
+    return IMPL
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Connection(object):
+    """Base class for storage system connections."""
+
+    @abc.abstractmethod
+    def __init__(self):
+        """Constructor."""

nimble/db/migration.py

@@ -0,0 +1,52 @@
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Database setup and migration commands."""
+
+from oslo_config import cfg
+from stevedore import driver
+
+_IMPL = None
+
+
+def get_backend():
+    global _IMPL
+    if not _IMPL:
+        cfg.CONF.import_opt('backend', 'oslo_db.options', group='database')
+        _IMPL = driver.DriverManager("nimble.database.migration_backend",
+                                     cfg.CONF.database.backend).driver
+    return _IMPL
+
+
+def upgrade(version=None):
+    """Migrate the database to `version` or the most recent version."""
+    return get_backend().upgrade(version)
+
+
+def version():
+    return get_backend().version()
+
+
+def stamp(version):
+    return get_backend().stamp(version)
+
+
+def revision(message, autogenerate):
+    return get_backend().revision(message, autogenerate)
+
+
+def create_schema():
+    return get_backend().create_schema()

nimble/db/sqlalchemy/alembic.ini

@@ -0,0 +1,54 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = %(here)s/alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+#sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

nimble/db/sqlalchemy/alembic/README

@@ -0,0 +1,12 @@
+Please see https://alembic.readthedocs.org/en/latest/index.html for general documentation
+
+To create alembic migrations use:
+$ nimble-dbsync revision --message --autogenerate
+
+Stamp db with most recent migration version, without actually running migrations
+$ nimble-dbsync stamp --revision head
+
+Upgrade can be performed by:
+$ nimble-dbsync - for backward compatibility
+$ nimble-dbsync upgrade
+# nimble-dbsync upgrade --revision head

nimble/db/sqlalchemy/alembic/env.py

@@ -0,0 +1,61 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from logging import config as log_config
+
+from alembic import context
+from oslo_db.sqlalchemy import enginefacade
+
+try:
+    # NOTE(whaom): This is to register the DB2 alembic code which
+    # is an optional runtime dependency.
+    from ibm_db_alembic.ibm_db import IbmDbImpl  # noqa
+except ImportError:
+    pass
+
+from nimble.db.sqlalchemy import models
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+log_config.fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+target_metadata = models.Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    engine = enginefacade.get_legacy_facade().get_engine()
+    with engine.connect() as connection:
+        context.configure(connection=connection,
+                          target_metadata=target_metadata)
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+run_migrations_online()

nimble/db/sqlalchemy/alembic/script.py.mako

@@ -0,0 +1,18 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}

nimble/db/sqlalchemy/api.py

@@ -0,0 +1,59 @@
+# Copyright 2016 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""SQLAlchemy storage backend."""
+
+import threading
+
+from oslo_db.sqlalchemy import enginefacade
+from oslo_log import log
+
+from nimble.db import api
+
+LOG = log.getLogger(__name__)
+
+
+_CONTEXT = threading.local()
+
+
+def get_backend():
+    """The backend is this module itself."""
+    return Connection()
+
+
+def _session_for_read():
+    return enginefacade.reader.using(_CONTEXT)
+
+
+def _session_for_write():
+    return enginefacade.writer.using(_CONTEXT)
+
+
+def model_query(model, *args, **kwargs):
+    """Query helper for simpler session usage.
+
+    :param session: if present, the session to use
+    """
+
+    with _session_for_read() as session:
+        query = session.query(model, *args)
+        return query
+
+
+class Connection(api.Connection):
+    """SqlAlchemy connection."""
+
+    def __init__(self):
+        pass

nimble/db/sqlalchemy/migration.py

@@ -0,0 +1,113 @@
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+import alembic
+from alembic import config as alembic_config
+import alembic.migration as alembic_migration
+from oslo_db import exception as db_exc
+from oslo_db.sqlalchemy import enginefacade
+
+from nimble.db.sqlalchemy import models
+
+
+def _alembic_config():
+    path = os.path.join(os.path.dirname(__file__), 'alembic.ini')
+    config = alembic_config.Config(path)
+    return config
+
+
+def version(config=None, engine=None):
+    """Current database version.
+
+    :returns: Database version
+    :rtype: string
+    """
+    if engine is None:
+        engine = enginefacade.get_legacy_facade().get_engine()
+    with engine.connect() as conn:
+        context = alembic_migration.MigrationContext.configure(conn)
+        return context.get_current_revision()
+
+
+def upgrade(revision, config=None):
+    """Used for upgrading database.
+
+    :param version: Desired database version
+    :type version: string
+    """
+    revision = revision or 'head'
+    config = config or _alembic_config()
+
+    alembic.command.upgrade(config, revision or 'head')
+
+
+def create_schema(config=None, engine=None):
+    """Create database schema from models description.
+
+    Can be used for initial installation instead of upgrade('head').
+    """
+    if engine is None:
+        engine = enginefacade.get_legacy_facade().get_engine()
+
+    # NOTE(viktors): If we will use metadata.create_all() for non empty db
+    #                schema, it will only add the new tables, but leave
+    #                existing as is. So we should avoid of this situation.
+    if version(engine=engine) is not None:
+        raise db_exc.DbMigrationError("DB schema is already under version"
+                                      " control. Use upgrade() instead")
+
+    models.Base.metadata.create_all(engine)
+    stamp('head', config=config)
+
+
+def downgrade(revision, config=None):
+    """Used for downgrading database.
+
+    :param version: Desired database version
+    :type version: string
+    """
+    revision = revision or 'base'
+    config = config or _alembic_config()
+    return alembic.command.downgrade(config, revision)
+
+
+def stamp(revision, config=None):
+    """Stamps database with provided revision.
+
+    Don't run any migrations.
+
+    :param revision: Should match one from repository or head - to stamp
+                     database with most recent revision
+    :type revision: string
+    """
+    config = config or _alembic_config()
+    return alembic.command.stamp(config, revision=revision)
+
+
+def revision(message=None, autogenerate=False, config=None):
+    """Creates template for migration.
+
+    :param message: Text that will be used for migration title
+    :type message: string
+    :param autogenerate: If True - generates diff based on current database
+                         state
+    :type autogenerate: bool
+    """
+    config = config or _alembic_config()
+    return alembic.command.revision(config, message=message,
+                                    autogenerate=autogenerate)

nimble/db/sqlalchemy/models.py

@@ -0,0 +1,54 @@
+# Copyright 2016 Huawei Technologies Co.,LTD.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+SQLAlchemy models for baremetal compute service.
+"""
+
+from oslo_db import options as db_options
+from oslo_db.sqlalchemy import models
+import six.moves.urllib.parse as urlparse
+from sqlalchemy.ext.declarative import declarative_base
+
+from nimble.common import paths
+from nimble.conf import CONF
+
+_DEFAULT_SQL_CONNECTION = 'sqlite:///' + paths.state_path_def('nimble.sqlite')
+
+
+db_options.set_defaults(CONF, _DEFAULT_SQL_CONNECTION, 'nimble.sqlite')
+
+
+def table_args():
+    engine_name = urlparse.urlparse(CONF.database.connection).scheme
+    if engine_name == 'mysql':
+        return {'mysql_engine': CONF.database.mysql_engine,
+                'mysql_charset': "utf8"}
+    return None
+
+
+class NimbleBase(models.TimestampMixin,
+                 models.ModelBase):
+
+    metadata = None
+
+    def as_dict(self):
+        d = {}
+        for c in self.__table__.columns:
+            d[c.name] = self[c.name]
+        return d
+
+
+Base = declarative_base(cls=NimbleBase)

requirements.txt

@@ -3,6 +3,8 @@
 # process, which may cause wedges in the gate later.
 
 pbr>=1.6 # Apache-2.0
+SQLAlchemy<1.1.0,>=1.0.10 # MIT
+alembic>=0.8.4 # MIT
 eventlet!=0.18.3,>=0.18.2 # MIT
 WebOb>=1.2.3 # MIT
 oslo.concurrency>=3.8.0 # Apache-2.0

setup.cfg

@@ -26,6 +26,7 @@ packages =
 [entry_points]
 console_scripts =
     nimble-api = nimble.cmd.api:main
+    nimble-dbsync = nimble.cmd.dbsync:main
 
 [build_sphinx]
 source-dir = doc/source