From bb2546f0ebda22049cc3b3d0ed23d763c09d979b Mon Sep 17 00:00:00 2001 From: Steven Dake Date: Tue, 2 Dec 2014 01:09:31 -0700 Subject: [PATCH] Copy Ironic's database model codebase The Ironic codebase is pretty simple for database access. This work leads into the introduction of versioned object technology from nova and ironic that will be entering oslo: https://review.openstack.org/#/c/127532/ This will drastically speed up the process of implementing moving the RPC objects across the network. Change-Id: I38aa451b658b66f5b6f10ced03ea2e0355af4ecd --- MANIFEST.in | 3 + magnum/cmd/db_manage.py | 100 +++ magnum/db/__init__.py | 0 magnum/db/api.py | 470 +++++++++++ magnum/db/migration.py | 56 ++ magnum/db/sqlalchemy/__init__.py | 0 magnum/db/sqlalchemy/alembic.ini | 54 ++ magnum/db/sqlalchemy/alembic/env.py | 54 ++ magnum/db/sqlalchemy/alembic/script.py.mako | 22 + .../2581ebaf0cb2_initial_migration.py | 83 ++ magnum/db/sqlalchemy/api.py | 784 ++++++++++++++++++ magnum/db/sqlalchemy/migration.py | 113 +++ magnum/db/sqlalchemy/models.py | 161 ++++ magnum/objects/sqlalchemy/__init__.py | 2 +- setup.cfg | 1 + 15 files changed, 1902 insertions(+), 1 deletion(-) create mode 100644 magnum/cmd/db_manage.py create mode 100644 magnum/db/__init__.py create mode 100644 magnum/db/api.py create mode 100644 magnum/db/migration.py create mode 100644 magnum/db/sqlalchemy/__init__.py create mode 100644 magnum/db/sqlalchemy/alembic.ini create mode 100644 magnum/db/sqlalchemy/alembic/env.py create mode 100644 magnum/db/sqlalchemy/alembic/script.py.mako create mode 100644 magnum/db/sqlalchemy/alembic/versions/2581ebaf0cb2_initial_migration.py create mode 100644 magnum/db/sqlalchemy/api.py create mode 100644 magnum/db/sqlalchemy/migration.py create mode 100644 magnum/db/sqlalchemy/models.py diff --git a/MANIFEST.in b/MANIFEST.in index c978a52dae..fa9d1afceb 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,4 +3,7 @@ include ChangeLog exclude .gitignore exclude .gitreview +include magnum/db/sqlalchemy/alembic.ini +include magnum/db/sqlalchemy/alembic/script.py.mako + global-exclude *.pyc diff --git a/magnum/cmd/db_manage.py b/magnum/cmd/db_manage.py new file mode 100644 index 0000000000..68ce3561de --- /dev/null +++ b/magnum/cmd/db_manage.py @@ -0,0 +1,100 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Starter script for magnum-db-manage.""" + +import os + +from oslo.config import cfg +from oslo.db import options +from oslo.db.sqlalchemy.migration_cli import manager + +from magnum.openstack.common import log as logging + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +def do_version(mgr): + print('Current DB revision is %s' % mgr.version()) + + +def do_upgrade(mgr): + mgr.upgrade(CONF.command.revision) + + +def do_downgrade(mgr): + mgr.downgrade(CONF.command.revision) + + +def do_stamp(mgr): + mgr.stamp(CONF.command.revision) + + +def do_revision(mgr): + mgr.revision(message=CONF.command.message, + autogenerate=CONF.command.autogenerate) + + +def add_command_parsers(subparsers): + parser = subparsers.add_parser('version') + parser.set_defaults(func=do_version) + + parser = subparsers.add_parser('upgrade') + parser.add_argument('revision', nargs='?') + parser.set_defaults(func=do_upgrade) + + parser = subparsers.add_parser('downgrade') + parser.add_argument('revision', nargs='?') + parser.set_defaults(func=do_downgrade) + + parser = subparsers.add_parser('stamp') + parser.add_argument('revision', nargs='?') + parser.set_defaults(func=do_stamp) + + parser = subparsers.add_parser('revision') + parser.add_argument('-m', '--message') + parser.add_argument('--autogenerate', action='store_true') + parser.set_defaults(func=do_revision) + + +def get_manager(): + if cfg.CONF.database.connection is None: + raise ValueError( + 'Database connection not set in /etc/magnum/magnum.conf') + + alembic_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), + '..', 'db', 'sqlalchemy', 'alembic.ini')) + migrate_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), + '..', 'db', 'sqlalchemy', 'alembic')) + migration_config = {'alembic_ini_path': alembic_path, + 'alembic_repo_path': migrate_path, + 'db_url': CONF.database.connection} + return manager.MigrationManager(migration_config) + + +def main(): + command_opt = cfg.SubCommandOpt('command', + title='Command', + help='Available commands', + handler=add_command_parsers) + CONF.register_cli_opt(command_opt) + + # set_defaults() is called to register the db options. + options.set_defaults(CONF) + + print ('manager is %s' % get_manager()) + CONF(project='magnum') + CONF.command.func(get_manager()) diff --git a/magnum/db/__init__.py b/magnum/db/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/magnum/db/api.py b/magnum/db/api.py new file mode 100644 index 0000000000..0f1b21d772 --- /dev/null +++ b/magnum/db/api.py @@ -0,0 +1,470 @@ +# -*- encoding: utf-8 -*- +# +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Base classes for storage engines +""" + +import abc + +from oslo.config import cfg +from oslo.db import api as db_api +import six + + +_BACKEND_MAPPING = {'sqlalchemy': 'magnum.db.sqlalchemy.api'} +IMPL = db_api.DBAPI.from_config(cfg.CONF, backend_mapping=_BACKEND_MAPPING, + lazy=True) + + +def get_instance(): + """Return a DB API instance.""" + return IMPL + + +@six.add_metaclass(abc.ABCMeta) +class Connection(object): + """Base class for storage system connections.""" + + @abc.abstractmethod + def __init__(self): + """Constructor.""" + + @abc.abstractmethod + def get_bay_list(self, columns=None, filters=None, limit=None, + marker=None, sort_key=None, sort_dir=None): + """Get specific columns for matching bays. + + Return a list of the specified columns for all bays that match the + specified filters. + + :param columns: List of column names to return. + Defaults to 'id' column when columns == None. + :param filters: Filters to apply. Defaults to None. + + :param limit: Maximum number of bays to return. + :param marker: the last item of the previous page; we return the next + result set. + :param sort_key: Attribute by which results should be sorted. + :param sort_dir: direction in which results should be sorted. + (asc, desc) + :returns: A list of tuples of the specified columns. + """ + + @abc.abstractmethod + def reserve_bay(self, tag, bay_id): + """Reserve a bay. + + To prevent other ManagerServices from manipulating the given + Bay while a Task is performed, mark it reserved by this host. + + :param tag: A string uniquely identifying the reservation holder. + :param bay_id: A bay id or uuid. + :returns: A Bay object. + :raises: BayNotFound if the bay is not found. + :raises: BayLocked if the bay is already reserved. + """ + + @abc.abstractmethod + def release_bay(self, tag, bay_id): + """Release the reservation on a bay. + + :param tag: A string uniquely identifying the reservation holder. + :param bay_id: A bay id or uuid. + :raises: BayNotFound if the bay is not found. + :raises: BayLocked if the bay is reserved by another host. + :raises: BayNotLocked if the bay was found to not have a + reservation at all. + """ + + @abc.abstractmethod + def create_bay(self, values): + """Create a new bay. + + :param values: A dict containing several items used to identify + and track the bay, and several dicts which are passed + into the Drivers when managing this bay. For example: + + :: + + { + 'uuid': utils.generate_uuid(), + 'name': 'example', + 'type': 'virt' + } + :returns: A bay. + """ + + @abc.abstractmethod + def get_bay_by_id(self, bay_id): + """Return a bay. + + :param bay_id: The id of a bay. + :returns: A bay. + """ + + @abc.abstractmethod + def get_bay_by_uuid(self, bay_uuid): + """Return a bay. + + :param bay_uuid: The uuid of a bay. + :returns: A bay. + """ + + @abc.abstractmethod + def get_bay_by_instance(self, instance): + """Return a bay. + + :param instance: The instance name or uuid to search for. + :returns: A bay. + """ + + @abc.abstractmethod + def destroy_bay(self, bay_id): + """Destroy a bay and all associated interfaces. + + :param bay_id: The id or uuid of a bay. + """ + + @abc.abstractmethod + def update_bay(self, bay_id, values): + """Update properties of a bay. + + :param bay_id: The id or uuid of a bay. + :returns: A bay. + :raises: BayAssociated + :raises: BayNotFound + """ + + @abc.abstractmethod + def get_container_list(self, columns=None, filters=None, limit=None, + marker=None, sort_key=None, sort_dir=None): + """Get specific columns for matching containers. + + Return a list of the specified columns for all containers that match + the specified filters. + + :param columns: List of column names to return. + Defaults to 'id' column when columns == None. + :param filters: Filters to apply. Defaults to None. + + :param limit: Maximum number of containers to return. + :param marker: the last item of the previous page; we return the next + result set. + :param sort_key: Attribute by which results should be sorted. + :param sort_dir: direction in which results should be sorted. + (asc, desc) + :returns: A list of tuples of the specified columns. + """ + + @abc.abstractmethod + def reserve_container(self, tag, container_id): + """Reserve a container. + + To prevent other ManagerServices from manipulating the given + Bay while a Task is performed, mark it reserved by this host. + + :param tag: A string uniquely identifying the reservation holder. + :param container_id: A container id or uuid. + :returns: A Bay object. + :raises: BayNotFound if the container is not found. + :raises: BayLocked if the container is already reserved. + """ + + @abc.abstractmethod + def release_container(self, tag, container_id): + """Release the reservation on a container. + + :param tag: A string uniquely identifying the reservation holder. + :param container_id: A container id or uuid. + :raises: BayNotFound if the container is not found. + :raises: BayLocked if the container is reserved by another host. + :raises: BayNotLocked if the container was found to not have a + reservation at all. + """ + + @abc.abstractmethod + def create_container(self, values): + """Create a new container. + + :param values: A dict containing several items used to identify + and track the container, and several dicts which are + passed + into the Drivers when managing this container. For + example: + + :: + + { + 'uuid': utils.generate_uuid(), + 'name': 'example', + 'type': 'virt' + } + :returns: A container. + """ + + @abc.abstractmethod + def get_container_by_id(self, container_id): + """Return a container. + + :param container_id: The id of a container. + :returns: A container. + """ + + @abc.abstractmethod + def get_container_by_uuid(self, container_uuid): + """Return a container. + + :param container_uuid: The uuid of a container. + :returns: A container. + """ + + @abc.abstractmethod + def get_container_by_instance(self, instance): + """Return a container. + + :param instance: The instance name or uuid to search for. + :returns: A container. + """ + + @abc.abstractmethod + def destroy_container(self, container_id): + """Destroy a container and all associated interfaces. + + :param container_id: The id or uuid of a container. + """ + + @abc.abstractmethod + def update_container(self, container_id, values): + """Update properties of a container. + + :param container_id: The id or uuid of a container. + :returns: A container. + :raises: BayAssociated + :raises: BayNotFound + """ + + @abc.abstractmethod + def get_pod_list(self, columns=None, filters=None, limit=None, + marker=None, sort_key=None, sort_dir=None): + """Get specific columns for matching pods. + + Return a list of the specified columns for all pods that match the + specified filters. + + :param columns: List of column names to return. + Defaults to 'id' column when columns == None. + :param filters: Filters to apply. Defaults to None. + + :param limit: Maximum number of pods to return. + :param marker: the last item of the previous page; we return the next + result set. + :param sort_key: Attribute by which results should be sorted. + :param sort_dir: direction in which results should be sorted. + (asc, desc) + :returns: A list of tuples of the specified columns. + """ + + @abc.abstractmethod + def reserve_pod(self, tag, pod_id): + """Reserve a pod. + + To prevent other ManagerServices from manipulating the given + Bay while a Task is performed, mark it reserved by this host. + + :param tag: A string uniquely identifying the reservation holder. + :param pod_id: A pod id or uuid. + :returns: A Bay object. + :raises: BayNotFound if the pod is not found. + :raises: BayLocked if the pod is already reserved. + """ + + @abc.abstractmethod + def release_pod(self, tag, pod_id): + """Release the reservation on a pod. + + :param tag: A string uniquely identifying the reservation holder. + :param pod_id: A pod id or uuid. + :raises: BayNotFound if the pod is not found. + :raises: BayLocked if the pod is reserved by another host. + :raises: BayNotLocked if the pod was found to not have a + reservation at all. + """ + + @abc.abstractmethod + def create_pod(self, values): + """Create a new pod. + + :param values: A dict containing several items used to identify + and track the pod, and several dicts which are passed + into the Drivers when managing this pod. For example: + + :: + + { + 'uuid': utils.generate_uuid(), + 'name': 'example', + 'type': 'virt' + } + :returns: A pod. + """ + + @abc.abstractmethod + def get_pod_by_id(self, pod_id): + """Return a pod. + + :param pod_id: The id of a pod. + :returns: A pod. + """ + + @abc.abstractmethod + def get_pod_by_uuid(self, pod_uuid): + """Return a pod. + + :param pod_uuid: The uuid of a pod. + :returns: A pod. + """ + + @abc.abstractmethod + def get_pod_by_instance(self, instance): + """Return a pod. + + :param instance: The instance name or uuid to search for. + :returns: A pod. + """ + + @abc.abstractmethod + def destroy_pod(self, pod_id): + """Destroy a pod and all associated interfaces. + + :param pod_id: The id or uuid of a pod. + """ + + @abc.abstractmethod + def update_pod(self, pod_id, values): + """Update properties of a pod. + + :param pod_id: The id or uuid of a pod. + :returns: A pod. + :raises: BayAssociated + :raises: BayNotFound + """ + + @abc.abstractmethod + def get_service_list(self, columns=None, filters=None, limit=None, + marker=None, sort_key=None, sort_dir=None): + """Get specific columns for matching services. + + Return a list of the specified columns for all services that match the + specified filters. + + :param columns: List of column names to return. + Defaults to 'id' column when columns == None. + :param filters: Filters to apply. Defaults to None. + + :param limit: Maximum number of services to return. + :param marker: the last item of the previous page; we return the next + result set. + :param sort_key: Attribute by which results should be sorted. + :param sort_dir: direction in which results should be sorted. + (asc, desc) + :returns: A list of tuples of the specified columns. + """ + + @abc.abstractmethod + def reserve_service(self, tag, service_id): + """Reserve a service. + + To prevent other ManagerServices from manipulating the given + Bay while a Task is performed, mark it reserved by this host. + + :param tag: A string uniquely identifying the reservation holder. + :param service_id: A service id or uuid. + :returns: A Bay object. + :raises: BayNotFound if the service is not found. + :raises: BayLocked if the service is already reserved. + """ + + @abc.abstractmethod + def release_service(self, tag, service_id): + """Release the reservation on a service. + + :param tag: A string uniquely identifying the reservation holder. + :param service_id: A service id or uuid. + :raises: BayNotFound if the service is not found. + :raises: BayLocked if the service is reserved by another host. + :raises: BayNotLocked if the service was found to not have a + reservation at all. + """ + + @abc.abstractmethod + def create_service(self, values): + """Create a new service. + + :param values: A dict containing several items used to identify + and track the service, and several dicts which are + passed into the Drivers when managing this service. + For example: + + :: + + { + 'uuid': utils.generate_uuid(), + 'name': 'example', + 'type': 'virt' + } + :returns: A service. + """ + + @abc.abstractmethod + def get_service_by_id(self, service_id): + """Return a service. + + :param service_id: The id of a service. + :returns: A service. + """ + + @abc.abstractmethod + def get_service_by_uuid(self, service_uuid): + """Return a service. + + :param service_uuid: The uuid of a service. + :returns: A service. + """ + + @abc.abstractmethod + def get_service_by_instance(self, instance): + """Return a service. + + :param instance: The instance name or uuid to search for. + :returns: A service. + """ + + @abc.abstractmethod + def destroy_service(self, service_id): + """Destroy a service and all associated interfaces. + + :param service_id: The id or uuid of a service. + """ + + @abc.abstractmethod + def update_service(self, service_id, values): + """Update properties of a service. + + :param service_id: The id or uuid of a service. + :returns: A service. + :raises: BayAssociated + :raises: BayNotFound + """ diff --git a/magnum/db/migration.py b/magnum/db/migration.py new file mode 100644 index 0000000000..ba294c1cc3 --- /dev/null +++ b/magnum/db/migration.py @@ -0,0 +1,56 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Database setup and migration commands.""" + +from oslo.config import cfg +from stevedore import driver + +_IMPL = None + + +def get_backend(): + global _IMPL + if not _IMPL: + cfg.CONF.import_opt('backend', 'oslo.db.options', group='database') + _IMPL = driver.DriverManager("magnum.database.migration_backend", + cfg.CONF.database.backend).driver + return _IMPL + + +def upgrade(version=None): + """Migrate the database to `version` or the most recent version.""" + return get_backend().upgrade(version) + + +def downgrade(version=None): + return get_backend().downgrade(version) + + +def version(): + return get_backend().version() + + +def stamp(version): + return get_backend().stamp(version) + + +def revision(message, autogenerate): + return get_backend().revision(message, autogenerate) + + +def create_schema(): + return get_backend().create_schema() diff --git a/magnum/db/sqlalchemy/__init__.py b/magnum/db/sqlalchemy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/magnum/db/sqlalchemy/alembic.ini b/magnum/db/sqlalchemy/alembic.ini new file mode 100644 index 0000000000..a768980345 --- /dev/null +++ b/magnum/db/sqlalchemy/alembic.ini @@ -0,0 +1,54 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s/alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +#sqlalchemy.url = driver://user:pass@localhost/dbname + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/magnum/db/sqlalchemy/alembic/env.py b/magnum/db/sqlalchemy/alembic/env.py new file mode 100644 index 0000000000..ff264b7652 --- /dev/null +++ b/magnum/db/sqlalchemy/alembic/env.py @@ -0,0 +1,54 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from logging import config as log_config + +from alembic import context + +from magnum.db.sqlalchemy import api as sqla_api +from magnum.db.sqlalchemy import models + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +log_config.fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +target_metadata = models.Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + engine = sqla_api.get_engine() + with engine.connect() as connection: + context.configure(connection=connection, + target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +run_migrations_online() diff --git a/magnum/db/sqlalchemy/alembic/script.py.mako b/magnum/db/sqlalchemy/alembic/script.py.mako new file mode 100644 index 0000000000..95702017ea --- /dev/null +++ b/magnum/db/sqlalchemy/alembic/script.py.mako @@ -0,0 +1,22 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/magnum/db/sqlalchemy/alembic/versions/2581ebaf0cb2_initial_migration.py b/magnum/db/sqlalchemy/alembic/versions/2581ebaf0cb2_initial_migration.py new file mode 100644 index 0000000000..6072e7df98 --- /dev/null +++ b/magnum/db/sqlalchemy/alembic/versions/2581ebaf0cb2_initial_migration.py @@ -0,0 +1,83 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""initial migration + +Revision ID: 2581ebaf0cb2 +Revises: None +Create Date: 2014-01-17 12:14:07.754448 + +""" + +# revision identifiers, used by Alembic. +revision = '2581ebaf0cb2' +down_revision = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # commands auto generated by Alembic - please adjust! + op.create_table( + 'bay', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uuid', sa.String(length=36), nullable=True), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('type', sa.String(length=20), nullable=True), + sa.PrimaryKeyConstraint('id'), + mysql_ENGINE='InnoDB', + mysql_DEFAULT_CHARSET='UTF8' + ) + op.create_table( + 'container', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id'), + mysql_ENGINE='InnoDB', + mysql_DEFAULT_CHARSET='UTF8' + ) + op.create_table( + 'pod', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uuid', sa.String(length=36), nullable=True), + sa.PrimaryKeyConstraint('id'), + mysql_ENGINE='InnoDB', + mysql_DEFAULT_CHARSET='UTF8' + ) + op.create_table( + 'service', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uuid', sa.String(length=36), nullable=True), + sa.PrimaryKeyConstraint('id'), + mysql_ENGINE='InnoDB', + mysql_DEFAULT_CHARSET='UTF8' + ) + # end Alembic commands + + +def downgrade(): + op.drop_table('bay') + op.drop_table('container') + op.drop_table('service') + op.drop_table('pod') +# We should probably remove the drops later ;-) +# raise NotImplementedError(('Downgrade from initial migration is' +# ' unsupported.')) diff --git a/magnum/db/sqlalchemy/api.py b/magnum/db/sqlalchemy/api.py new file mode 100644 index 0000000000..8fc87d05b8 --- /dev/null +++ b/magnum/db/sqlalchemy/api.py @@ -0,0 +1,784 @@ +# -*- encoding: utf-8 -*- +# +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""SQLAlchemy storage backend.""" + +import datetime + +from oslo.config import cfg +from oslo.db import exception as db_exc +from oslo.db.sqlalchemy import session as db_session +from oslo.db.sqlalchemy import utils as db_utils +from oslo.utils import timeutils +from sqlalchemy.orm.exc import NoResultFound + +from magnum.common import exception +from magnum.common import utils +from magnum.db import api +from magnum.db.sqlalchemy import models +from magnum.openstack.common._i18n import _ +from magnum.openstack.common import log + +CONF = cfg.CONF + +LOG = log.getLogger(__name__) + + +_FACADE = None + + +def _create_facade_lazily(): + global _FACADE + if _FACADE is None: + _FACADE = db_session.EngineFacade.from_config(CONF) + return _FACADE + + +def get_engine(): + facade = _create_facade_lazily() + return facade.get_engine() + + +def get_session(**kwargs): + facade = _create_facade_lazily() + return facade.get_session(**kwargs) + + +def get_backend(): + """The backend is this module itself.""" + return Connection() + + +def model_query(model, *args, **kwargs): + """Query helper for simpler session usage. + + :param session: if present, the session to use + """ + + session = kwargs.get('session') or get_session() + query = session.query(model, *args) + return query + + +def add_identity_filter(query, value): + """Adds an identity filter to a query. + + Filters results by ID, if supplied value is a valid integer. + Otherwise attempts to filter results by UUID. + + :param query: Initial query to add filter to. + :param value: Value for filtering results by. + :return: Modified query. + """ + if utils.is_int_like(value): + return query.filter_by(id=value) + elif utils.is_uuid_like(value): + return query.filter_by(uuid=value) + else: + raise exception.InvalidIdentity(identity=value) + + +def _check_port_change_forbidden(port, session): + bay_id = port['bay_id'] + if bay_id is not None: + query = model_query(models.Bay, session=session) + query = query.filter_by(id=bay_id) + bay_ref = query.one() + if bay_ref['reservation'] is not None: + raise exception.BayLocked(bay=bay_ref['uuid'], + host=bay_ref['reservation']) + + +def _paginate_query(model, limit=None, marker=None, sort_key=None, + sort_dir=None, query=None): + if not query: + query = model_query(model) + sort_keys = ['id'] + if sort_key and sort_key not in sort_keys: + sort_keys.insert(0, sort_key) + query = db_utils.paginate_query(query, model, limit, sort_keys, + marker=marker, sort_dir=sort_dir) + return query.all() + + +class Connection(api.Connection): + """SqlAlchemy connection.""" + + def __init__(self): + pass + + def _add_bays_filters(self, query, filters): + if filters is None: + filters = [] + + if 'associated' in filters: + if filters['associated']: + query = query.filter(models.Bay.instance_uuid is not None) + else: + query = query.filter(models.Bay.instance_uuid is None) + if 'reserved' in filters: + if filters['reserved']: + query = query.filter(models.Bay.reservation is not None) + else: + query = query.filter(models.Bay.reservation is None) + if 'maintenance' in filters: + query = query.filter_by(maintenance=filters['maintenance']) + if 'driver' in filters: + query = query.filter_by(driver=filters['driver']) + if 'provision_state' in filters: + query = query.filter_by(provision_state=filters['provision_state']) + if 'provisioned_before' in filters: + limit = timeutils.utcnow() - datetime.timedelta( + seconds=filters['provisioned_before']) + query = query.filter(models.Bay.provision_updated_at < limit) + + return query + + def get_bayinfo_list(self, columns=None, filters=None, limit=None, + marker=None, sort_key=None, sort_dir=None): + # list-ify columns default values because it is bad form + # to include a mutable list in function definitions. + if columns is None: + columns = [models.Bay.id] + else: + columns = [getattr(models.Bay, c) for c in columns] + + query = model_query(*columns, base_model=models.Bay) + query = self._add_bays_filters(query, filters) + return _paginate_query(models.Bay, limit, marker, + sort_key, sort_dir, query) + + def get_bay_list(self, filters=None, limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.Bay) + query = self._add_bays_filters(query, filters) + return _paginate_query(models.Bay, limit, marker, + sort_key, sort_dir, query) + + def reserve_bay(self, tag, bay_id): + session = get_session() + with session.begin(): + query = model_query(models.Bay, session=session) + query = add_identity_filter(query, bay_id) + # be optimistic and assume we usually create a reservation + count = query.filter_by(reservation=None).update( + {'reservation': tag}, synchronize_session=False) + try: + bay = query.one() + if count != 1: + # Nothing updated and bay exists. Must already be + # locked. + raise exception.BayLocked(bay=bay_id, + host=bay['reservation']) + return bay + except NoResultFound: + raise exception.BayNotFound(bay_id) + + def release_bay(self, tag, bay_id): + session = get_session() + with session.begin(): + query = model_query(models.Bay, session=session) + query = add_identity_filter(query, bay_id) + # be optimistic and assume we usually release a reservation + count = query.filter_by(reservation=tag).update( + {'reservation': None}, synchronize_session=False) + try: + if count != 1: + bay = query.one() + if bay['reservation'] is None: + raise exception.BayNotLocked(bay=bay_id) + else: + raise exception.BayLocked(bay=bay_id, + host=bay['reservation']) + except NoResultFound: + raise exception.BayNotFound(bay_id) + + def create_bay(self, values): + # ensure defaults are present for new bays + if not values.get('uuid'): + values['uuid'] = utils.generate_uuid() + + bay = models.Bay() + bay.update(values) + try: + bay.save() + except db_exc.DBDuplicateEntry as exc: + if 'instance_uuid' in exc.columns: + raise exception.InstanceAssociated( + instance_uuid=values['instance_uuid'], + bay=values['uuid']) + raise exception.BayAlreadyExists(uuid=values['uuid']) + return bay + + def get_bay_by_id(self, bay_id): + query = model_query(models.Bay).filter_by(id=bay_id) + try: + return query.one() + except NoResultFound: + raise exception.BayNotFound(bay=bay_id) + + def get_bay_by_uuid(self, bay_uuid): + query = model_query(models.Bay).filter_by(uuid=bay_uuid) + try: + return query.one() + except NoResultFound: + raise exception.BayNotFound(bay=bay_uuid) + + def get_bay_by_instance(self, instance): + if not utils.is_uuid_like(instance): + raise exception.InvalidUUID(uuid=instance) + + query = (model_query(models.Bay) + .filter_by(instance_uuid=instance)) + + try: + result = query.one() + except NoResultFound: + raise exception.InstanceNotFound(instance=instance) + + return result + + def destroy_bay(self, bay_id): + session = get_session() + with session.begin(): + query = model_query(models.Bay, session=session) + query = add_identity_filter(query, bay_id) + query.delete() + + def update_bay(self, bay_id, values): + # NOTE(dtantsur): this can lead to very strange errors + if 'uuid' in values: + msg = _("Cannot overwrite UUID for an existing Bay.") + raise exception.InvalidParameterValue(err=msg) + + try: + return self._do_update_bay(bay_id, values) + except db_exc.DBDuplicateEntry: + raise exception.InstanceAssociated( + instance_uuid=values['instance_uuid'], + bay=bay_id) + + def _do_update_bay(self, bay_id, values): + session = get_session() + with session.begin(): + query = model_query(models.Bay, session=session) + query = add_identity_filter(query, bay_id) + try: + ref = query.with_lockmode('update').one() + except NoResultFound: + raise exception.BayNotFound(bay=bay_id) + + # Prevent instance_uuid overwriting + if values.get("instance_uuid") and ref.instance_uuid: + raise exception.BayAssociated(bay=bay_id, + instance=ref.instance_uuid) + + if 'provision_state' in values: + values['provision_updated_at'] = timeutils.utcnow() + + ref.update(values) + return ref + + def _add_containers_filters(self, query, filters): + if filters is None: + filters = [] + + if 'associated' in filters: + if filters['associated']: + query = query.filter(models.Container.instance_uuid is not + None) + else: + query = query.filter(models.Container.instance_uuid is None) + if 'reserved' in filters: + if filters['reserved']: + query = query.filter(models.Container.reservation is not None) + else: + query = query.filter(models.Container.reservation is None) + if 'maintenance' in filters: + query = query.filter_by(maintenance=filters['maintenance']) + if 'driver' in filters: + query = query.filter_by(driver=filters['driver']) + if 'provision_state' in filters: + query = query.filter_by(provision_state=filters['provision_state']) + if 'provisioned_before' in filters: + limit = timeutils.utcnow() - datetime.timedelta( + seconds=filters['provisioned_before']) + query = query.filter(models.Container.provision_updated_at < limit) + + return query + + def get_containerinfo_list(self, columns=None, filters=None, limit=None, + marker=None, sort_key=None, sort_dir=None): + # list-ify columns default values because it is bad form + # to include a mutable list in function definitions. + if columns is None: + columns = [models.Container.id] + else: + columns = [getattr(models.Container, c) for c in columns] + + query = model_query(*columns, base_model=models.Container) + query = self._add_containers_filters(query, filters) + return _paginate_query(models.Container, limit, marker, + sort_key, sort_dir, query) + + def get_container_list(self, filters=None, limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.Container) + query = self._add_containers_filters(query, filters) + return _paginate_query(models.Container, limit, marker, + sort_key, sort_dir, query) + + def reserve_container(self, tag, container_id): + session = get_session() + with session.begin(): + query = model_query(models.Container, session=session) + query = add_identity_filter(query, container_id) + # be optimistic and assume we usually create a reservation + count = query.filter_by(reservation=None).update( + {'reservation': tag}, synchronize_session=False) + try: + container = query.one() + if count != 1: + # Nothing updated and container exists. Must already be + # locked. + raise exception.ContainerLocked(container=container_id, + host=container['reservation']) + return container + except NoResultFound: + raise exception.ContainerNotFound(container_id) + + def release_container(self, tag, container_id): + session = get_session() + with session.begin(): + query = model_query(models.Container, session=session) + query = add_identity_filter(query, container_id) + # be optimistic and assume we usually release a reservation + count = query.filter_by(reservation=tag).update( + {'reservation': None}, synchronize_session=False) + try: + if count != 1: + container = query.one() + if container['reservation'] is None: + raise exception.ContainerNotLocked( + container=container_id) + else: + raise exception.ContainerLocked(container=container_id, + host=container['reservation']) + except NoResultFound: + raise exception.ContainerNotFound(container_id) + + def create_container(self, values): + # ensure defaults are present for new containers + if not values.get('uuid'): + values['uuid'] = utils.generate_uuid() + + container = models.Container() + container.update(values) + try: + container.save() + except db_exc.DBDuplicateEntry as exc: + if 'instance_uuid' in exc.columns: + raise exception.InstanceAssociated( + instance_uuid=values['instance_uuid'], + container=values['uuid']) + raise exception.ContainerAlreadyExists(uuid=values['uuid']) + return container + + def get_container_by_id(self, container_id): + query = model_query(models.Container).filter_by(id=container_id) + try: + return query.one() + except NoResultFound: + raise exception.ContainerNotFound(container=container_id) + + def get_container_by_uuid(self, container_uuid): + query = model_query(models.Container).filter_by(uuid=container_uuid) + try: + return query.one() + except NoResultFound: + raise exception.ContainerNotFound(container=container_uuid) + + def get_container_by_instance(self, instance): + if not utils.is_uuid_like(instance): + raise exception.InvalidUUID(uuid=instance) + + query = (model_query(models.Container) + .filter_by(instance_uuid=instance)) + + try: + result = query.one() + except NoResultFound: + raise exception.InstanceNotFound(instance=instance) + + return result + + def destroy_container(self, container_id): + session = get_session() + with session.begin(): + query = model_query(models.Container, session=session) + query = add_identity_filter(query, container_id) + query.delete() + + def update_container(self, container_id, values): + # NOTE(dtantsur): this can lead to very strange errors + if 'uuid' in values: + msg = _("Cannot overwrite UUID for an existing Container.") + raise exception.InvalidParameterValue(err=msg) + + try: + return self._do_update_container(container_id, values) + except db_exc.DBDuplicateEntry: + raise exception.InstanceAssociated( + instance_uuid=values['instance_uuid'], + container=container_id) + + def _do_update_container(self, container_id, values): + session = get_session() + with session.begin(): + query = model_query(models.Container, session=session) + query = add_identity_filter(query, container_id) + try: + ref = query.with_lockmode('update').one() + except NoResultFound: + raise exception.ContainerNotFound(container=container_id) + + # Prevent instance_uuid overwriting + if values.get("instance_uuid") and ref.instance_uuid: + raise exception.ContainerAssociated(container=container_id, + instance=ref.instance_uuid) + + if 'provision_state' in values: + values['provision_updated_at'] = timeutils.utcnow() + + ref.update(values) + return ref + + def get_podinfo_list(self, columns=None, filters=None, limit=None, + marker=None, sort_key=None, sort_dir=None): + # list-ify columns default values because it is bad form + # to include a mutable list in function definitions. + if columns is None: + columns = [models.Pod.id] + else: + columns = [getattr(models.Pod, c) for c in columns] + + query = model_query(*columns, base_model=models.Pod) + query = self._add_pods_filters(query, filters) + return _paginate_query(models.Pod, limit, marker, + sort_key, sort_dir, query) + + def get_pod_list(self, filters=None, limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.Pod) + query = self._add_pods_filters(query, filters) + return _paginate_query(models.Pod, limit, marker, + sort_key, sort_dir, query) + + def reserve_pod(self, tag, pod_id): + session = get_session() + with session.begin(): + query = model_query(models.Pod, session=session) + query = add_identity_filter(query, pod_id) + # be optimistic and assume we usually create a reservation + count = query.filter_by(reservation=None).update( + {'reservation': tag}, synchronize_session=False) + try: + pod = query.one() + if count != 1: + # Nothing updated and pod exists. Must already be + # locked. + raise exception.PodLocked(pod=pod_id, + host=pod['reservation']) + return pod + except NoResultFound: + raise exception.PodNotFound(pod_id) + + def release_pod(self, tag, pod_id): + session = get_session() + with session.begin(): + query = model_query(models.Pod, session=session) + query = add_identity_filter(query, pod_id) + # be optimistic and assume we usually release a reservation + count = query.filter_by(reservation=tag).update( + {'reservation': None}, synchronize_session=False) + try: + if count != 1: + pod = query.one() + if pod['reservation'] is None: + raise exception.PodNotLocked(pod=pod_id) + else: + raise exception.PodLocked(pod=pod_id, + host=pod['reservation']) + except NoResultFound: + raise exception.PodNotFound(pod_id) + + def create_pod(self, values): + # ensure defaults are present for new pods + if not values.get('uuid'): + values['uuid'] = utils.generate_uuid() + + pod = models.Pod() + pod.update(values) + try: + pod.save() + except db_exc.DBDuplicateEntry as exc: + if 'instance_uuid' in exc.columns: + raise exception.InstanceAssociated( + instance_uuid=values['instance_uuid'], + pod=values['uuid']) + raise exception.PodAlreadyExists(uuid=values['uuid']) + return pod + + def get_pod_by_id(self, pod_id): + query = model_query(models.Pod).filter_by(id=pod_id) + try: + return query.one() + except NoResultFound: + raise exception.PodNotFound(pod=pod_id) + + def get_pod_by_uuid(self, pod_uuid): + query = model_query(models.Pod).filter_by(uuid=pod_uuid) + try: + return query.one() + except NoResultFound: + raise exception.PodNotFound(pod=pod_uuid) + + def get_pod_by_instance(self, instance): + if not utils.is_uuid_like(instance): + raise exception.InvalidUUID(uuid=instance) + + query = (model_query(models.Pod) + .filter_by(instance_uuid=instance)) + + try: + result = query.one() + except NoResultFound: + raise exception.InstanceNotFound(instance=instance) + + return result + + def destroy_pod(self, pod_id): + session = get_session() + with session.begin(): + query = model_query(models.Pod, session=session) + query = add_identity_filter(query, pod_id) + query.delete() + + def update_pod(self, pod_id, values): + # NOTE(dtantsur): this can lead to very strange errors + if 'uuid' in values: + msg = _("Cannot overwrite UUID for an existing Pod.") + raise exception.InvalidParameterValue(err=msg) + + try: + return self._do_update_pod(pod_id, values) + except db_exc.DBDuplicateEntry: + raise exception.InstanceAssociated( + instance_uuid=values['instance_uuid'], + pod=pod_id) + + def _do_update_pod(self, pod_id, values): + session = get_session() + with session.begin(): + query = model_query(models.Pod, session=session) + query = add_identity_filter(query, pod_id) + try: + ref = query.with_lockmode('update').one() + except NoResultFound: + raise exception.PodNotFound(pod=pod_id) + + # Prevent instance_uuid overwriting + if values.get("instance_uuid") and ref.instance_uuid: + raise exception.PodAssociated(pod=pod_id, + instance=ref.instance_uuid) + + if 'provision_state' in values: + values['provision_updated_at'] = timeutils.utcnow() + + ref.update(values) + return ref + + def _add_services_filters(self, query, filters): + if filters is None: + filters = [] + + if 'associated' in filters: + if filters['associated']: + query = query.filter(models.Service.instance_uuid is not None) + else: + query = query.filter(models.Service.instance_uuid is None) + if 'reserved' in filters: + if filters['reserved']: + query = query.filter(models.Service.reservation is not None) + else: + query = query.filter(models.Service.reservation is None) + if 'maintenance' in filters: + query = query.filter_by(maintenance=filters['maintenance']) + if 'driver' in filters: + query = query.filter_by(driver=filters['driver']) + if 'provision_state' in filters: + query = query.filter_by(provision_state=filters['provision_state']) + if 'provisioned_before' in filters: + limit = timeutils.utcnow() - datetime.timedelta( + seconds=filters['provisioned_before']) + query = query.filter(models.Service.provision_updated_at < limit) + + return query + + def get_serviceinfo_list(self, columns=None, filters=None, limit=None, + marker=None, sort_key=None, sort_dir=None): + # list-ify columns default values because it is bad form + # to include a mutable list in function definitions. + if columns is None: + columns = [models.Service.id] + else: + columns = [getattr(models.Service, c) for c in columns] + + query = model_query(*columns, base_model=models.Service) + query = self._add_services_filters(query, filters) + return _paginate_query(models.Service, limit, marker, + sort_key, sort_dir, query) + + def get_service_list(self, filters=None, limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.Service) + query = self._add_services_filters(query, filters) + return _paginate_query(models.Service, limit, marker, + sort_key, sort_dir, query) + + def reserve_service(self, tag, service_id): + session = get_session() + with session.begin(): + query = model_query(models.Service, session=session) + query = add_identity_filter(query, service_id) + # be optimistic and assume we usually create a reservation + count = query.filter_by(reservation=None).update( + {'reservation': tag}, synchronize_session=False) + try: + service = query.one() + if count != 1: + # Nothing updated and service exists. Must already be + # locked. + raise exception.ServiceLocked(service=service_id, + host=service['reservation']) + return service + except NoResultFound: + raise exception.ServiceNotFound(service_id) + + def release_service(self, tag, service_id): + session = get_session() + with session.begin(): + query = model_query(models.Service, session=session) + query = add_identity_filter(query, service_id) + # be optimistic and assume we usually release a reservation + count = query.filter_by(reservation=tag).update( + {'reservation': None}, synchronize_session=False) + try: + if count != 1: + service = query.one() + if service['reservation'] is None: + raise exception.ServiceNotLocked(service=service_id) + else: + raise exception.ServiceLocked(service=service_id, + host=service['reservation']) + except NoResultFound: + raise exception.ServiceNotFound(service_id) + + def create_service(self, values): + # ensure defaults are present for new services + if not values.get('uuid'): + values['uuid'] = utils.generate_uuid() + + service = models.Service() + service.update(values) + try: + service.save() + except db_exc.DBDuplicateEntry as exc: + if 'instance_uuid' in exc.columns: + raise exception.InstanceAssociated( + instance_uuid=values['instance_uuid'], + service=values['uuid']) + raise exception.ServiceAlreadyExists(uuid=values['uuid']) + return service + + def get_service_by_id(self, service_id): + query = model_query(models.Service).filter_by(id=service_id) + try: + return query.one() + except NoResultFound: + raise exception.ServiceNotFound(service=service_id) + + def get_service_by_uuid(self, service_uuid): + query = model_query(models.Service).filter_by(uuid=service_uuid) + try: + return query.one() + except NoResultFound: + raise exception.ServiceNotFound(service=service_uuid) + + def get_service_by_instance(self, instance): + if not utils.is_uuid_like(instance): + raise exception.InvalidUUID(uuid=instance) + + query = (model_query(models.Service) + .filter_by(instance_uuid=instance)) + + try: + result = query.one() + except NoResultFound: + raise exception.InstanceNotFound(instance=instance) + + return result + + def destroy_service(self, service_id): + session = get_session() + with session.begin(): + query = model_query(models.Service, session=session) + query = add_identity_filter(query, service_id) + query.delete() + + def update_service(self, service_id, values): + # NOTE(dtantsur): this can lead to very strange errors + if 'uuid' in values: + msg = _("Cannot overwrite UUID for an existing Service.") + raise exception.InvalidParameterValue(err=msg) + + try: + return self._do_update_service(service_id, values) + except db_exc.DBDuplicateEntry: + raise exception.InstanceAssociated( + instance_uuid=values['instance_uuid'], + service=service_id) + + def _do_update_service(self, service_id, values): + session = get_session() + with session.begin(): + query = model_query(models.Service, session=session) + query = add_identity_filter(query, service_id) + try: + ref = query.with_lockmode('update').one() + except NoResultFound: + raise exception.ServiceNotFound(service=service_id) + + # Prevent instance_uuid overwriting + if values.get("instance_uuid") and ref.instance_uuid: + raise exception.ServiceAssociated(service=service_id, + instance=ref.instance_uuid) + + if 'provision_state' in values: + values['provision_updated_at'] = timeutils.utcnow() + + ref.update(values) + return ref diff --git a/magnum/db/sqlalchemy/migration.py b/magnum/db/sqlalchemy/migration.py new file mode 100644 index 0000000000..f2bb1e97b5 --- /dev/null +++ b/magnum/db/sqlalchemy/migration.py @@ -0,0 +1,113 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +import alembic +from alembic import config as alembic_config +import alembic.migration as alembic_migration +from oslo.db import exception as db_exc + +from magnum.db.sqlalchemy import api as sqla_api +from magnum.db.sqlalchemy import models + + +def _alembic_config(): + path = os.path.join(os.path.dirname(__file__), 'alembic.ini') + config = alembic_config.Config(path) + return config + + +def version(config=None, engine=None): + """Current database version. + + :returns: Database version + :rtype: string + """ + if engine is None: + engine = sqla_api.get_engine() + with engine.connect() as conn: + context = alembic_migration.MigrationContext.configure(conn) + return context.get_current_revision() + + +def upgrade(revision, config=None): + """Used for upgrading database. + + :param version: Desired database version + :type version: string + """ + revision = revision or 'head' + config = config or _alembic_config() + + alembic.command.upgrade(config, revision or 'head') + + +def create_schema(config=None, engine=None): + """Create database schema from models description. + + Can be used for initial installation instead of upgrade('head'). + """ + if engine is None: + engine = sqla_api.get_engine() + + # NOTE(viktors): If we will use metadata.create_all() for non empty db + # schema, it will only add the new tables, but leave + # existing as is. So we should avoid of this situation. + if version(engine=engine) is not None: + raise db_exc.DbMigrationError("DB schema is already under version" + " control. Use upgrade() instead") + + models.Base.metadata.create_all(engine) + stamp('head', config=config) + + +def downgrade(revision, config=None): + """Used for downgrading database. + + :param version: Desired database version + :type version: string + """ + revision = revision or 'base' + config = config or _alembic_config() + return alembic.command.downgrade(config, revision) + + +def stamp(revision, config=None): + """Stamps database with provided revision. + + Don't run any migrations. + + :param revision: Should match one from repository or head - to stamp + database with most recent revision + :type revision: string + """ + config = config or _alembic_config() + return alembic.command.stamp(config, revision=revision) + + +def revision(message=None, autogenerate=False, config=None): + """Creates template for migration. + + :param message: Text that will be used for migration title + :type message: string + :param autogenerate: If True - generates diff based on current database + state + :type autogenerate: bool + """ + config = config or _alembic_config() + return alembic.command.revision(config, message=message, + autogenerate=autogenerate) diff --git a/magnum/db/sqlalchemy/models.py b/magnum/db/sqlalchemy/models.py new file mode 100644 index 0000000000..523b0f8ce6 --- /dev/null +++ b/magnum/db/sqlalchemy/models.py @@ -0,0 +1,161 @@ +# -*- encoding: utf-8 -*- +# +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +SQLAlchemy models for container service +""" + +import json + +from oslo.config import cfg +from oslo.db import options as db_options +from oslo.db.sqlalchemy import models +import six.moves.urllib.parse as urlparse +from sqlalchemy import Column +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import Integer +from sqlalchemy import schema +from sqlalchemy import String +from sqlalchemy.types import TypeDecorator, TEXT + +from magnum.common import paths + + +sql_opts = [ + cfg.StrOpt('mysql_engine', + default='InnoDB', + help='MySQL engine to use.') +] + +_DEFAULT_SQL_CONNECTION = 'sqlite:///' + paths.state_path_def('magnum.sqlite') + + +cfg.CONF.register_opts(sql_opts, 'database') +db_options.set_defaults(cfg.CONF, _DEFAULT_SQL_CONNECTION, 'magnum.sqlite') + + +def table_args(): + engine_name = urlparse.urlparse(cfg.CONF.database.connection).scheme + if engine_name == 'mysql': + return {'mysql_engine': cfg.CONF.database.mysql_engine, + 'mysql_charset': "utf8"} + return None + + +class JsonEncodedType(TypeDecorator): + """Abstract base type serialized as json-encoded string in db.""" + type = None + impl = TEXT + + def process_bind_param(self, value, dialect): + if value is None: + # Save default value according to current type to keep the + # interface the consistent. + value = self.type() + elif not isinstance(value, self.type): + raise TypeError("%s supposes to store %s objects, but %s given" + % (self.__class__.__name__, + self.type.__name__, + type(value).__name__)) + serialized_value = json.dumps(value) + return serialized_value + + def process_result_value(self, value, dialect): + if value is not None: + value = json.loads(value) + return value + + +class JSONEncodedDict(JsonEncodedType): + """Represents dict serialized as json-encoded string in db.""" + type = dict + + +class JSONEncodedList(JsonEncodedType): + """Represents list serialized as json-encoded string in db.""" + type = list + + +class MagnumBase(models.TimestampMixin, + models.ModelBase): + + metadata = None + + def as_dict(self): + d = {} + for c in self.__table__.columns: + d[c.name] = self[c.name] + return d + + def save(self, session=None): + import magnum.db.sqlalchemy.api as db_api + + if session is None: + session = db_api.get_session() + + super(MagnumBase, self).save(session) + +Base = declarative_base(cls=MagnumBase) + + +class Bay(Base): + """Represents a bay.""" + + __tablename__ = 'bay' + __table_args__ = ( + schema.UniqueConstraint('uuid', name='uniq_bay0uuid'), + table_args() + ) + id = Column(Integer, primary_key=True) + uuid = Column(String(36)) + name = Column(String(255)) + type = Column(String(20)) + + +class Container(Base): + """Represents a container.""" + + __tablename__ = 'container' + __table_args__ = ( + schema.UniqueConstraint('uuid', name='uniq_container0uuid'), + table_args() + ) + id = Column(Integer, primary_key=True) + uuid = Column(String(36)) + + +class Pod(Base): + """Represents a pod.""" + + __tablename__ = 'pod' + __table_args__ = ( + schema.UniqueConstraint('uuid', name='uniq_pod0uuid'), + table_args() + ) + id = Column(Integer, primary_key=True) + uuid = Column(String(36)) + + +class AbrviceObject(Base): + """Represents a software service.""" + + __tablename__ = 'service' + __table_args__ = ( + schema.UniqueConstraint('uuid', name='uniq_service0uuid'), + table_args() + ) + id = Column(Integer, primary_key=True) + uuid = Column(String(36)) diff --git a/magnum/objects/sqlalchemy/__init__.py b/magnum/objects/sqlalchemy/__init__.py index 00cdc481c8..0f8fec6689 100644 --- a/magnum/objects/sqlalchemy/__init__.py +++ b/magnum/objects/sqlalchemy/__init__.py @@ -58,4 +58,4 @@ def load(): objects.registry.add(abstract_bay.BayList, bay.BayList) objects.registry.add(abstract_container.Container, container.Container) objects.registry.add(abstract_container.ContainerList, - container.ContainerList) \ No newline at end of file + container.ContainerList) diff --git a/setup.cfg b/setup.cfg index 148629e3cb..925bf732bf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,6 +48,7 @@ console_scripts = magnum-api = magnum.cmd.api:main magnum-conductor = magnum.cmd.conductor:main magnum-backend = magnum.cmd.backend:main + magnum-db-manage = magnum.cmd.db_manage:main oslo.config.opts = magnum = magnum.opts:list_opts