diff --git a/doc/source/db.rst b/doc/source/db.rst index a57052a0..770565e7 100644 --- a/doc/source/db.rst +++ b/doc/source/db.rst @@ -29,9 +29,10 @@ The commands should be executed as a subcommand of 'db': Sync the Database ----------------- - glance-manage db sync + glance-manage db sync -Place a database under migration control and upgrade, creating it first if necessary. +Place an existing database under migration control and upgrade it to the +specified VERSION. Determining the Database Version diff --git a/doc/source/man/glancemanage.rst b/doc/source/man/glancemanage.rst index 621bfcad..0ecd2895 100644 --- a/doc/source/man/glancemanage.rst +++ b/doc/source/man/glancemanage.rst @@ -53,9 +53,9 @@ COMMANDS **db_version_control** Place the database under migration control. - **db_sync ** - Place a database under migration control and upgrade, creating - it first if necessary. + **db_sync ** + Place an existing database under migration control and upgrade it to + the specified VERSION. **db_export_metadefs [PATH | PREFIX]** Export the metadata definitions into json format. By default the @@ -80,10 +80,6 @@ OPTIONS .. include:: general_options.rst - **--sql_connection=CONN_STRING** - A proper SQLAlchemy connection string as described - `here `_ - .. include:: footer.rst CONFIGURATION diff --git a/glance/cmd/manage.py b/glance/cmd/manage.py index 16e08c52..aa2ec04c 100644 --- a/glance/cmd/manage.py +++ b/glance/cmd/manage.py @@ -39,8 +39,9 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): sys.path.insert(0, possible_topdir) +from alembic import command as alembic_command + from oslo_config import cfg -from oslo_db.sqlalchemy import migration from oslo_log import log as logging from oslo_utils import encodeutils import six @@ -49,6 +50,7 @@ from glance.common import config from glance.common import exception from glance import context from glance.db import migration as db_migration +from glance.db.sqlalchemy import alembic_migrations from glance.db.sqlalchemy import api as db_api from glance.db.sqlalchemy import metadata from glance.i18n import _ @@ -73,39 +75,56 @@ class DbCommands(object): def version(self): """Print database's current migration level""" - print(migration.db_version(db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, - db_migration.INIT_VERSION)) + current_heads = alembic_migrations.get_current_alembic_heads() + if current_heads: + # Migrations are managed by alembic + for head in current_heads: + print(head) + else: + # Migrations are managed by legacy versioning scheme + print(_('Database is either not under migration control or under ' + 'legacy migration control, please run ' + '"glance-manage db sync" to place the database under ' + 'alembic migration control.')) @args('--version', metavar='', help='Database version') - def upgrade(self, version=None): + def upgrade(self, version='heads'): """Upgrade the database's migration level""" - migration.db_sync(db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, - version) + self.sync(version) @args('--version', metavar='', help='Database version') - def version_control(self, version=None): + def version_control(self, version=db_migration.ALEMBIC_INIT_VERSION): """Place a database under migration control""" - migration.db_version_control(db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, - version) + + if version is None: + version = db_migration.ALEMBIC_INIT_VERSION + + a_config = alembic_migrations.get_alembic_config() + alembic_command.stamp(a_config, version) + print(_("Placed database under migration control at " + "revision:"), version) @args('--version', metavar='', help='Database version') - @args('--current_version', metavar='', - help='Current Database version') - def sync(self, version=None, current_version=None): + def sync(self, version='heads'): """ - Place a database under migration control and upgrade it, - creating first if necessary. + Place an existing database under migration control and upgrade it. """ - if current_version not in (None, 'None'): - migration.db_version_control(db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, - version=current_version) - migration.db_sync(db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, - version) + if version is None: + version = 'heads' + + alembic_migrations.place_database_under_alembic_control() + + a_config = alembic_migrations.get_alembic_config() + alembic_command.upgrade(a_config, version) + heads = alembic_migrations.get_current_alembic_heads() + if heads is None: + raise Exception("Database sync failed") + revs = ", ".join(heads) + if version is 'heads': + print(_("Upgraded database, current revision(s):"), revs) + else: + print(_('Upgraded database to: %(v)s, current revision(s): %(r)s') + % {'v': version, 'r': revs}) @args('--path', metavar='', help='Path to the directory or file ' 'where json metadata is stored') @@ -179,15 +198,14 @@ class DbLegacyCommands(object): def version(self): self.command_object.version() - def upgrade(self, version=None): + def upgrade(self, version='heads'): self.command_object.upgrade(CONF.command.version) - def version_control(self, version=None): + def version_control(self, version=db_migration.ALEMBIC_INIT_VERSION): self.command_object.version_control(CONF.command.version) - def sync(self, version=None, current_version=None): - self.command_object.sync(CONF.command.version, - CONF.command.current_version) + def sync(self, version='heads'): + self.command_object.sync(CONF.command.version) def load_metadefs(self, path=None, merge=False, prefer_new=False, overwrite=False): @@ -224,7 +242,6 @@ def add_legacy_command_parsers(command_object, subparsers): parser = subparsers.add_parser('db_sync') parser.set_defaults(action_fn=legacy_command_object.sync) parser.add_argument('version', nargs='?') - parser.add_argument('current_version', nargs='?') parser.set_defaults(action='db_sync') parser = subparsers.add_parser('db_load_metadefs') diff --git a/glance/db/migration.py b/glance/db/migration.py index 3709f068..a35591db 100644 --- a/glance/db/migration.py +++ b/glance/db/migration.py @@ -45,6 +45,7 @@ def get_backend(): cfg.CONF.database.backend).driver return _IMPL +ALEMBIC_INIT_VERSION = 'liberty' INIT_VERSION = 0 MIGRATE_REPO_PATH = os.path.join( diff --git a/glance/db/sqlalchemy/alembic_migrations/README b/glance/db/sqlalchemy/alembic_migrations/README new file mode 100644 index 00000000..2500aa1b --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. diff --git a/glance/db/sqlalchemy/alembic_migrations/__init__.py b/glance/db/sqlalchemy/alembic_migrations/__init__.py new file mode 100644 index 00000000..b1a0499b --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/__init__.py @@ -0,0 +1,99 @@ +# Copyright 2016 Rackspace +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys + +from alembic import command as alembic_command +from alembic import config as alembic_config +from alembic import migration as alembic_migration +from oslo_db import exception as db_exception +from oslo_db.sqlalchemy import migration + +from glance.db import migration as db_migration +from glance.db.sqlalchemy import api as db_api +from glance.i18n import _ + + +def get_alembic_config(): + """Return a valid alembic config object""" + ini_path = os.path.join(os.path.dirname(__file__), 'alembic.ini') + config = alembic_config.Config(os.path.abspath(ini_path)) + dbconn = str(db_api.get_engine().url) + config.set_main_option('sqlalchemy.url', dbconn) + return config + + +def get_current_alembic_heads(): + """Return current heads (if any) from the alembic migration table""" + engine = db_api.get_engine() + with engine.connect() as conn: + context = alembic_migration.MigrationContext.configure(conn) + heads = context.get_current_heads() + return heads + + +def get_current_legacy_head(): + try: + legacy_head = migration.db_version(db_api.get_engine(), + db_migration.MIGRATE_REPO_PATH, + db_migration.INIT_VERSION) + except db_exception.DbMigrationError: + legacy_head = None + return legacy_head + + +def is_database_under_alembic_control(): + if get_current_alembic_heads(): + return True + return False + + +def is_database_under_migrate_control(): + if get_current_legacy_head(): + return True + return False + + +def place_database_under_alembic_control(): + a_config = get_alembic_config() + + if not is_database_under_migrate_control(): + return + + if not is_database_under_alembic_control(): + print(_("Database is currently not under Alembic's migration " + "control.")) + head = get_current_legacy_head() + if head == 42: + alembic_version = 'liberty' + elif head == 43: + alembic_version = 'mitaka01' + elif head == 44: + alembic_version = 'mitaka02' + elif head == 45: + alembic_version = 'ocata01' + elif head in range(1, 42): + print("Legacy head: ", head) + sys.exit(_("The current database version is not supported any " + "more. Please upgrade to Liberty release first.")) + else: + sys.exit(_("Unable to place database under Alembic's migration " + "control. Unknown database state, can't proceed " + "further.")) + + print(_("Placing database under Alembic's migration control at " + "revision:"), alembic_version) + alembic_command.stamp(a_config, alembic_version) diff --git a/glance/db/sqlalchemy/alembic_migrations/add_artifacts_tables.py b/glance/db/sqlalchemy/alembic_migrations/add_artifacts_tables.py new file mode 100644 index 00000000..6d965f6a --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/add_artifacts_tables.py @@ -0,0 +1,224 @@ +# Copyright 2016 Rackspace +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from alembic import op +from sqlalchemy.schema import ( + Column, PrimaryKeyConstraint, ForeignKeyConstraint) + +from glance.db.sqlalchemy.migrate_repo.schema import ( + Boolean, DateTime, Integer, BigInteger, String, Text, Numeric) # noqa + + +def _add_artifacts_table(): + op.create_table('artifacts', + Column('id', String(length=36), nullable=False), + Column('name', String(length=255), nullable=False), + Column('type_name', String(length=255), nullable=False), + Column('type_version_prefix', + BigInteger(), + nullable=False), + Column('type_version_suffix', + String(length=255), + nullable=True), + Column('type_version_meta', + String(length=255), + nullable=True), + Column('version_prefix', BigInteger(), nullable=False), + Column('version_suffix', + String(length=255), + nullable=True), + Column('version_meta', String(length=255), nullable=True), + Column('description', Text(), nullable=True), + Column('visibility', String(length=32), nullable=False), + Column('state', String(length=32), nullable=False), + Column('owner', String(length=255), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=False), + Column('deleted_at', DateTime(), nullable=True), + Column('published_at', DateTime(), nullable=True), + PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_artifact_name_and_version', + 'artifacts', + ['name', 'version_prefix', 'version_suffix'], + unique=False) + op.create_index('ix_artifact_owner', 'artifacts', ['owner'], unique=False) + op.create_index('ix_artifact_state', 'artifacts', ['state'], unique=False) + op.create_index('ix_artifact_type', + 'artifacts', + ['type_name', + 'type_version_prefix', + 'type_version_suffix'], + unique=False) + op.create_index('ix_artifact_visibility', + 'artifacts', + ['visibility'], + unique=False) + + +def _add_artifact_blobs_table(): + op.create_table('artifact_blobs', + Column('id', String(length=36), nullable=False), + Column('artifact_id', String(length=36), nullable=False), + Column('size', BigInteger(), nullable=False), + Column('checksum', String(length=32), nullable=True), + Column('name', String(length=255), nullable=False), + Column('item_key', String(length=329), nullable=True), + Column('position', Integer(), nullable=True), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=False), + ForeignKeyConstraint(['artifact_id'], ['artifacts.id'], ), + PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_artifact_blobs_artifact_id', + 'artifact_blobs', + ['artifact_id'], + unique=False) + op.create_index('ix_artifact_blobs_name', + 'artifact_blobs', + ['name'], + unique=False) + + +def _add_artifact_dependencies_table(): + op.create_table('artifact_dependencies', + Column('id', String(length=36), nullable=False), + Column('artifact_source', + String(length=36), + nullable=False), + Column('artifact_dest', String(length=36), nullable=False), + Column('artifact_origin', + String(length=36), + nullable=False), + Column('is_direct', Boolean(), nullable=False), + Column('position', Integer(), nullable=True), + Column('name', String(length=36), nullable=True), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=False), + ForeignKeyConstraint(['artifact_dest'], + ['artifacts.id'], ), + ForeignKeyConstraint(['artifact_origin'], + ['artifacts.id'], ), + ForeignKeyConstraint(['artifact_source'], + ['artifacts.id'], ), + PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_artifact_dependencies_dest_id', + 'artifact_dependencies', + ['artifact_dest'], + unique=False) + op.create_index('ix_artifact_dependencies_direct_dependencies', + 'artifact_dependencies', + ['artifact_source', 'is_direct'], + unique=False) + op.create_index('ix_artifact_dependencies_origin_id', + 'artifact_dependencies', + ['artifact_origin'], + unique=False) + op.create_index('ix_artifact_dependencies_source_id', + 'artifact_dependencies', + ['artifact_source'], + unique=False) + + +def _add_artifact_properties_table(): + op.create_table('artifact_properties', + Column('id', String(length=36), nullable=False), + Column('artifact_id', String(length=36), nullable=False), + Column('name', String(length=255), nullable=False), + Column('string_value', String(length=255), nullable=True), + Column('int_value', Integer(), nullable=True), + Column('numeric_value', Numeric(), nullable=True), + Column('bool_value', Boolean(), nullable=True), + Column('text_value', Text(), nullable=True), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=False), + Column('position', Integer(), nullable=True), + ForeignKeyConstraint(['artifact_id'], ['artifacts.id'], ), + PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_artifact_properties_artifact_id', + 'artifact_properties', + ['artifact_id'], + unique=False) + op.create_index('ix_artifact_properties_name', + 'artifact_properties', + ['name'], + unique=False) + + +def _add_artifact_tags_table(): + op.create_table('artifact_tags', + Column('id', String(length=36), nullable=False), + Column('artifact_id', String(length=36), nullable=False), + Column('value', String(length=255), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=False), + ForeignKeyConstraint(['artifact_id'], ['artifacts.id'], ), + PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_artifact_tags_artifact_id', + 'artifact_tags', + ['artifact_id'], + unique=False) + op.create_index('ix_artifact_tags_artifact_id_tag_value', + 'artifact_tags', + ['artifact_id', 'value'], + unique=False) + + +def _add_artifact_blob_locations_table(): + op.create_table('artifact_blob_locations', + Column('id', String(length=36), nullable=False), + Column('blob_id', String(length=36), nullable=False), + Column('value', Text(), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=False), + Column('position', Integer(), nullable=True), + Column('status', String(length=36), nullable=True), + ForeignKeyConstraint(['blob_id'], ['artifact_blobs.id'], ), + PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_artifact_blob_locations_blob_id', + 'artifact_blob_locations', + ['blob_id'], + unique=False) + + +def upgrade(): + _add_artifacts_table() + _add_artifact_blobs_table() + _add_artifact_dependencies_table() + _add_artifact_properties_table() + _add_artifact_tags_table() + _add_artifact_blob_locations_table() diff --git a/glance/db/sqlalchemy/alembic_migrations/add_images_tables.py b/glance/db/sqlalchemy/alembic_migrations/add_images_tables.py new file mode 100644 index 00000000..399c77e4 --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/add_images_tables.py @@ -0,0 +1,201 @@ +# Copyright 2016 Rackspace +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from alembic import op +from sqlalchemy import sql +from sqlalchemy.schema import ( + Column, PrimaryKeyConstraint, ForeignKeyConstraint, UniqueConstraint) + +from glance.db.sqlalchemy.migrate_repo.schema import ( + Boolean, DateTime, Integer, BigInteger, String, Text) # noqa +from glance.db.sqlalchemy.models import JSONEncodedDict + + +def _add_images_table(): + op.create_table('images', + Column('id', String(length=36), nullable=False), + Column('name', String(length=255), nullable=True), + Column('size', BigInteger(), nullable=True), + Column('status', String(length=30), nullable=False), + Column('is_public', Boolean(), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + Column('deleted_at', DateTime(), nullable=True), + Column('deleted', Boolean(), nullable=False), + Column('disk_format', String(length=20), nullable=True), + Column('container_format', + String(length=20), + nullable=True), + Column('checksum', String(length=32), nullable=True), + Column('owner', String(length=255), nullable=True), + Column('min_disk', Integer(), nullable=False), + Column('min_ram', Integer(), nullable=False), + Column('protected', + Boolean(), + server_default=sql.false(), + nullable=False), + Column('virtual_size', BigInteger(), nullable=True), + PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('checksum_image_idx', + 'images', + ['checksum'], + unique=False) + op.create_index('ix_images_deleted', + 'images', + ['deleted'], + unique=False) + op.create_index('ix_images_is_public', + 'images', + ['is_public'], + unique=False) + op.create_index('owner_image_idx', + 'images', + ['owner'], + unique=False) + + +def _add_image_properties_table(): + op.create_table('image_properties', + Column('id', Integer(), nullable=False), + Column('image_id', String(length=36), nullable=False), + Column('name', String(length=255), nullable=False), + Column('value', Text(), nullable=True), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + Column('deleted_at', DateTime(), nullable=True), + Column('deleted', Boolean(), nullable=False), + PrimaryKeyConstraint('id'), + ForeignKeyConstraint(['image_id'], ['images.id'], ), + UniqueConstraint('image_id', + 'name', + name='ix_image_properties_image_id_name'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_image_properties_deleted', + 'image_properties', + ['deleted'], + unique=False) + op.create_index('ix_image_properties_image_id', + 'image_properties', + ['image_id'], + unique=False) + + +def _add_image_locations_table(): + op.create_table('image_locations', + Column('id', Integer(), nullable=False), + Column('image_id', String(length=36), nullable=False), + Column('value', Text(), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + Column('deleted_at', DateTime(), nullable=True), + Column('deleted', Boolean(), nullable=False), + Column('meta_data', JSONEncodedDict(), nullable=True), + Column('status', + String(length=30), + server_default='active', + nullable=False), + PrimaryKeyConstraint('id'), + ForeignKeyConstraint(['image_id'], ['images.id'], ), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_image_locations_deleted', + 'image_locations', + ['deleted'], + unique=False) + op.create_index('ix_image_locations_image_id', + 'image_locations', + ['image_id'], + unique=False) + + +def _add_image_members_table(): + deleted_member_constraint = 'image_members_image_id_member_deleted_at_key' + op.create_table('image_members', + Column('id', Integer(), nullable=False), + Column('image_id', String(length=36), nullable=False), + Column('member', String(length=255), nullable=False), + Column('can_share', Boolean(), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + Column('deleted_at', DateTime(), nullable=True), + Column('deleted', Boolean(), nullable=False), + Column('status', + String(length=20), + server_default='pending', + nullable=False), + ForeignKeyConstraint(['image_id'], ['images.id'], ), + PrimaryKeyConstraint('id'), + UniqueConstraint('image_id', + 'member', + 'deleted_at', + name=deleted_member_constraint), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_image_members_deleted', + 'image_members', + ['deleted'], + unique=False) + op.create_index('ix_image_members_image_id', + 'image_members', + ['image_id'], + unique=False) + op.create_index('ix_image_members_image_id_member', + 'image_members', + ['image_id', 'member'], + unique=False) + + +def _add_images_tags_table(): + op.create_table('image_tags', + Column('id', Integer(), nullable=False), + Column('image_id', String(length=36), nullable=False), + Column('value', String(length=255), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + Column('deleted_at', DateTime(), nullable=True), + Column('deleted', Boolean(), nullable=False), + ForeignKeyConstraint(['image_id'], ['images.id'], ), + PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_image_tags_image_id', + 'image_tags', + ['image_id'], + unique=False) + op.create_index('ix_image_tags_image_id_tag_value', + 'image_tags', + ['image_id', 'value'], + unique=False) + + +def upgrade(): + _add_images_table() + _add_image_properties_table() + _add_image_locations_table() + _add_image_members_table() + _add_images_tags_table() diff --git a/glance/db/sqlalchemy/alembic_migrations/add_metadefs_tables.py b/glance/db/sqlalchemy/alembic_migrations/add_metadefs_tables.py new file mode 100644 index 00000000..96fa7333 --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/add_metadefs_tables.py @@ -0,0 +1,171 @@ +# Copyright 2016 Rackspace +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from alembic import op +from sqlalchemy.schema import ( + Column, PrimaryKeyConstraint, ForeignKeyConstraint, UniqueConstraint) + +from glance.db.sqlalchemy.migrate_repo.schema import ( + Boolean, DateTime, Integer, String, Text) # noqa +from glance.db.sqlalchemy.models import JSONEncodedDict + + +def _add_metadef_namespaces_table(): + op.create_table('metadef_namespaces', + Column('id', Integer(), nullable=False), + Column('namespace', String(length=80), nullable=False), + Column('display_name', String(length=80), nullable=True), + Column('description', Text(), nullable=True), + Column('visibility', String(length=32), nullable=True), + Column('protected', Boolean(), nullable=True), + Column('owner', String(length=255), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + PrimaryKeyConstraint('id'), + UniqueConstraint('namespace', + name='uq_metadef_namespaces_namespace'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_metadef_namespaces_owner', + 'metadef_namespaces', + ['owner'], + unique=False) + + +def _add_metadef_resource_types_table(): + op.create_table('metadef_resource_types', + Column('id', Integer(), nullable=False), + Column('name', String(length=80), nullable=False), + Column('protected', Boolean(), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + PrimaryKeyConstraint('id'), + UniqueConstraint('name', + name='uq_metadef_resource_types_name'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + +def _add_metadef_namespace_resource_types_table(): + op.create_table('metadef_namespace_resource_types', + Column('resource_type_id', Integer(), nullable=False), + Column('namespace_id', Integer(), nullable=False), + Column('properties_target', + String(length=80), + nullable=True), + Column('prefix', String(length=80), nullable=True), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + ForeignKeyConstraint(['namespace_id'], + ['metadef_namespaces.id'], ), + ForeignKeyConstraint(['resource_type_id'], + ['metadef_resource_types.id'], ), + PrimaryKeyConstraint('resource_type_id', 'namespace_id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_metadef_ns_res_types_namespace_id', + 'metadef_namespace_resource_types', + ['namespace_id'], + unique=False) + + +def _add_metadef_objects_table(): + ns_id_name_constraint = 'uq_metadef_objects_namespace_id_name' + + op.create_table('metadef_objects', + Column('id', Integer(), nullable=False), + Column('namespace_id', Integer(), nullable=False), + Column('name', String(length=80), nullable=False), + Column('description', Text(), nullable=True), + Column('required', Text(), nullable=True), + Column('json_schema', JSONEncodedDict(), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + ForeignKeyConstraint(['namespace_id'], + ['metadef_namespaces.id'], ), + PrimaryKeyConstraint('id'), + UniqueConstraint('namespace_id', + 'name', + name=ns_id_name_constraint), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_metadef_objects_name', + 'metadef_objects', + ['name'], + unique=False) + + +def _add_metadef_properties_table(): + ns_id_name_constraint = 'uq_metadef_properties_namespace_id_name' + op.create_table('metadef_properties', + Column('id', Integer(), nullable=False), + Column('namespace_id', Integer(), nullable=False), + Column('name', String(length=80), nullable=False), + Column('json_schema', JSONEncodedDict(), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + ForeignKeyConstraint(['namespace_id'], + ['metadef_namespaces.id'], ), + PrimaryKeyConstraint('id'), + UniqueConstraint('namespace_id', + 'name', + name=ns_id_name_constraint), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_metadef_properties_name', + 'metadef_properties', + ['name'], + unique=False) + + +def _add_metadef_tags_table(): + op.create_table('metadef_tags', + Column('id', Integer(), nullable=False), + Column('namespace_id', Integer(), nullable=False), + Column('name', String(length=80), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + ForeignKeyConstraint(['namespace_id'], + ['metadef_namespaces.id'], ), + PrimaryKeyConstraint('id'), + UniqueConstraint('namespace_id', + 'name', + name='uq_metadef_tags_namespace_id_name'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_metadef_tags_name', + 'metadef_tags', + ['name'], + unique=False) + + +def upgrade(): + _add_metadef_namespaces_table() + _add_metadef_resource_types_table() + _add_metadef_namespace_resource_types_table() + _add_metadef_objects_table() + _add_metadef_properties_table() + _add_metadef_tags_table() diff --git a/glance/db/sqlalchemy/alembic_migrations/add_tasks_tables.py b/glance/db/sqlalchemy/alembic_migrations/add_tasks_tables.py new file mode 100644 index 00000000..d199557a --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/add_tasks_tables.py @@ -0,0 +1,66 @@ +# Copyright 2016 Rackspace +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from alembic import op +from sqlalchemy.schema import ( + Column, PrimaryKeyConstraint, ForeignKeyConstraint) + +from glance.db.sqlalchemy.migrate_repo.schema import ( + Boolean, DateTime, String, Text) # noqa +from glance.db.sqlalchemy.models import JSONEncodedDict + + +def _add_tasks_table(): + op.create_table('tasks', + Column('id', String(length=36), nullable=False), + Column('type', String(length=30), nullable=False), + Column('status', String(length=30), nullable=False), + Column('owner', String(length=255), nullable=False), + Column('expires_at', DateTime(), nullable=True), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + Column('deleted_at', DateTime(), nullable=True), + Column('deleted', Boolean(), nullable=False), + PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_tasks_deleted', 'tasks', ['deleted'], unique=False) + op.create_index('ix_tasks_owner', 'tasks', ['owner'], unique=False) + op.create_index('ix_tasks_status', 'tasks', ['status'], unique=False) + op.create_index('ix_tasks_type', 'tasks', ['type'], unique=False) + op.create_index('ix_tasks_updated_at', + 'tasks', + ['updated_at'], + unique=False) + + +def _add_task_info_table(): + op.create_table('task_info', + Column('task_id', String(length=36), nullable=False), + Column('input', JSONEncodedDict(), nullable=True), + Column('result', JSONEncodedDict(), nullable=True), + Column('message', Text(), nullable=True), + ForeignKeyConstraint(['task_id'], ['tasks.id'], ), + PrimaryKeyConstraint('task_id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + +def upgrade(): + _add_tasks_table() + _add_task_info_table() diff --git a/glance/db/sqlalchemy/alembic_migrations/alembic.ini b/glance/db/sqlalchemy/alembic_migrations/alembic.ini new file mode 100644 index 00000000..640a9af4 --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/alembic.ini @@ -0,0 +1,69 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to alembic_migrations/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat alembic_migrations/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# Uncomment and update to your sql connection string if wishing to run +# alembic directly from command line +#sqlalchemy.url = + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/glance/db/sqlalchemy/alembic_migrations/env.py b/glance/db/sqlalchemy/alembic_migrations/env.py new file mode 100644 index 00000000..0de0d82f --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/env.py @@ -0,0 +1,92 @@ +# Copyright 2016 Rackspace +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import with_statement +from logging import config as log_config + +from alembic import context +from sqlalchemy import engine_from_config, pool + +from glance.db.sqlalchemy import models +from glance.db.sqlalchemy import models_glare +from glance.db.sqlalchemy import models_metadef + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +log_config.fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +target_metadata = models.BASE.metadata +for table in models_glare.BASE.metadata.sorted_tables: + target_metadata._add_table(table.name, table.schema, table) +for table in models_metadef.BASE_DICT.metadata.sorted_tables: + target_metadata._add_table(table.name, table.schema, table) + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/glance/db/sqlalchemy/alembic_migrations/migrate.cfg b/glance/db/sqlalchemy/alembic_migrations/migrate.cfg new file mode 100644 index 00000000..8ddf0500 --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/migrate.cfg @@ -0,0 +1,20 @@ +[db_settings] +# Used to identify which repository this database is versioned under. +# You can use the name of your project. +repository_id=Glance Migrations + +# The name of the database table used to track the schema version. +# This name shouldn't already be used by your project. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. +version_table=alembic_version + +# When committing a change script, Migrate will attempt to generate the +# sql for all supported databases; normally, if one of them fails - probably +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually +# be using to ensure your updates to that database work properly. +# This must be a list; example: ['postgres','sqlite'] +required_dbs=[] diff --git a/glance/db/sqlalchemy/alembic_migrations/script.py.mako b/glance/db/sqlalchemy/alembic_migrations/script.py.mako new file mode 100644 index 00000000..8323caac --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/script.py.mako @@ -0,0 +1,20 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/liberty_initial.py b/glance/db/sqlalchemy/alembic_migrations/versions/liberty_initial.py new file mode 100644 index 00000000..2d56680e --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/liberty_initial.py @@ -0,0 +1,40 @@ +# Copyright 2016 Rackspace +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""liberty initial + +Revision ID: liberty +Revises: +Create Date: 2016-08-03 16:06:59.657433 + +""" + +from glance.db.sqlalchemy.alembic_migrations import add_artifacts_tables +from glance.db.sqlalchemy.alembic_migrations import add_images_tables +from glance.db.sqlalchemy.alembic_migrations import add_metadefs_tables +from glance.db.sqlalchemy.alembic_migrations import add_tasks_tables + +# revision identifiers, used by Alembic. +revision = 'liberty' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + add_images_tables.upgrade() + add_tasks_tables.upgrade() + add_metadefs_tables.upgrade() + add_artifacts_tables.upgrade() diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/mitaka01_add_image_created_updated_idx.py b/glance/db/sqlalchemy/alembic_migrations/versions/mitaka01_add_image_created_updated_idx.py new file mode 100644 index 00000000..5180c675 --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/mitaka01_add_image_created_updated_idx.py @@ -0,0 +1,47 @@ +# Copyright 2016 Rackspace +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""add index on created_at and updated_at columns of 'images' table + +Revision ID: mitaka01 +Revises: liberty +Create Date: 2016-08-03 17:19:35.306161 + +""" + +from alembic import op +from sqlalchemy import MetaData, Table, Index + + +# revision identifiers, used by Alembic. +revision = 'mitaka01' +down_revision = 'liberty' +branch_labels = None +depends_on = None + +CREATED_AT_INDEX = 'created_at_image_idx' +UPDATED_AT_INDEX = 'updated_at_image_idx' + + +def upgrade(): + migrate_engine = op.get_bind() + meta = MetaData(bind=migrate_engine) + + images = Table('images', meta, autoload=True) + + created_index = Index(CREATED_AT_INDEX, images.c.created_at) + created_index.create(migrate_engine) + updated_index = Index(UPDATED_AT_INDEX, images.c.updated_at) + updated_index.create(migrate_engine) diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/mitaka02_update_metadef_os_nova_server.py b/glance/db/sqlalchemy/alembic_migrations/versions/mitaka02_update_metadef_os_nova_server.py new file mode 100644 index 00000000..9416c68a --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/mitaka02_update_metadef_os_nova_server.py @@ -0,0 +1,42 @@ +# Copyright 2016 Rackspace +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""update metadef os_nova_server + +Revision ID: mitaka02 +Revises: mitaka01 +Create Date: 2016-08-03 17:23:23.041663 + +""" + +from alembic import op +from sqlalchemy import MetaData, Table + + +# revision identifiers, used by Alembic. +revision = 'mitaka02' +down_revision = 'mitaka01' +branch_labels = None +depends_on = None + + +def upgrade(): + migrate_engine = op.get_bind() + meta = MetaData(bind=migrate_engine) + + resource_types_table = Table('metadef_resource_types', meta, autoload=True) + + resource_types_table.update(values={'name': 'OS::Nova::Server'}).where( + resource_types_table.c.name == 'OS::Nova::Instance').execute() diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.py b/glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.py new file mode 100644 index 00000000..5d66513e --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.py @@ -0,0 +1,72 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""add visibility to and remove is_public from images + +Revision ID: ocata01 +Revises: mitaka02 +Create Date: 2017-01-20 12:58:16.647499 + +""" + +import os + +from alembic import op +from sqlalchemy import Column, Enum, MetaData, select, Table, not_, and_ +import sqlparse + +# revision identifiers, used by Alembic. +revision = 'ocata01' +down_revision = 'mitaka02' +branch_labels = None +depends_on = None + + +def upgrade(): + migrate_engine = op.get_bind() + meta = MetaData(bind=migrate_engine) + + engine_name = migrate_engine.engine.name + if engine_name == 'sqlite': + sql_file = os.path.splitext(__file__)[0] + sql_file += '.sql' + with open(sql_file, 'r') as sqlite_script: + sql = sqlparse.format(sqlite_script.read(), strip_comments=True) + for statement in sqlparse.split(sql): + op.execute(statement) + return + + enum = Enum('private', 'public', 'shared', 'community', metadata=meta, + name='image_visibility') + enum.create() + v_col = Column('visibility', enum, nullable=False, server_default='shared') + op.add_column('images', v_col) + + op.create_index('visibility_image_idx', 'images', ['visibility']) + + images = Table('images', meta, autoload=True) + images.update(values={'visibility': 'public'}).where( + images.c.is_public).execute() + + image_members = Table('image_members', meta, autoload=True) + + # NOTE(dharinic): Mark all the non-public images as 'private' first + images.update().values(visibility='private').where( + not_(images.c.is_public)).execute() + # NOTE(dharinic): Identify 'shared' images from the above + images.update().values(visibility='shared').where(and_( + images.c.visibility == 'private', images.c.id.in_(select( + [image_members.c.image_id]).distinct().where( + not_(image_members.c.deleted))))).execute() + + op.drop_index('ix_images_is_public', 'images') + op.drop_column('images', 'is_public') diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.sql b/glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.sql new file mode 100644 index 00000000..0e848cce --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.sql @@ -0,0 +1,162 @@ +CREATE TEMPORARY TABLE images_backup ( + id VARCHAR(36) NOT NULL, + name VARCHAR(255), + size INTEGER, + status VARCHAR(30) NOT NULL, + is_public BOOLEAN NOT NULL, + created_at DATETIME NOT NULL, + updated_at DATETIME, + deleted_at DATETIME, + deleted BOOLEAN NOT NULL, + disk_format VARCHAR(20), + container_format VARCHAR(20), + checksum VARCHAR(32), + owner VARCHAR(255), + min_disk INTEGER NOT NULL, + min_ram INTEGER NOT NULL, + protected BOOLEAN DEFAULT 0 NOT NULL, + virtual_size INTEGER, + PRIMARY KEY (id), + CHECK (is_public IN (0, 1)), + CHECK (deleted IN (0, 1)), + CHECK (protected IN (0, 1)) +); + +INSERT INTO images_backup + SELECT id, + name, + size, + status, + is_public, + created_at, + updated_at, + deleted_at, + deleted, + disk_format, + container_format, + checksum, + owner, + min_disk, + min_ram, + protected, + virtual_size + FROM images; + +DROP TABLE images; + +CREATE TABLE images ( + id VARCHAR(36) NOT NULL, + name VARCHAR(255), + size INTEGER, + status VARCHAR(30) NOT NULL, + created_at DATETIME NOT NULL, + updated_at DATETIME, + deleted_at DATETIME, + deleted BOOLEAN NOT NULL, + disk_format VARCHAR(20), + container_format VARCHAR(20), + checksum VARCHAR(32), + owner VARCHAR(255), + min_disk INTEGER NOT NULL, + min_ram INTEGER NOT NULL, + protected BOOLEAN DEFAULT 0 NOT NULL, + virtual_size INTEGER, + visibility VARCHAR(9) DEFAULT 'shared' NOT NULL, + PRIMARY KEY (id), + CHECK (deleted IN (0, 1)), + CHECK (protected IN (0, 1)), + CONSTRAINT image_visibility CHECK (visibility IN ('private', 'public', 'shared', 'community')) +); + +CREATE INDEX checksum_image_idx ON images (checksum); +CREATE INDEX visibility_image_idx ON images (visibility); +CREATE INDEX ix_images_deleted ON images (deleted); +CREATE INDEX owner_image_idx ON images (owner); +CREATE INDEX created_at_image_idx ON images (created_at); +CREATE INDEX updated_at_image_idx ON images (updated_at); + +-- Copy over all the 'public' rows + +INSERT INTO images ( + id, + name, + size, + status, + created_at, + updated_at, + deleted_at, + deleted, + disk_format, + container_format, + checksum, + owner, + min_disk, + min_ram, + protected, + virtual_size + ) + SELECT id, + name, + size, + status, + created_at, + updated_at, + deleted_at, + deleted, + disk_format, + container_format, + checksum, + owner, + min_disk, + min_ram, + protected, + virtual_size + FROM images_backup + WHERE is_public=1; + + +UPDATE images SET visibility='public'; + +-- Now copy over the 'private' rows + +INSERT INTO images ( + id, + name, + size, + status, + created_at, + updated_at, + deleted_at, + deleted, + disk_format, + container_format, + checksum, + owner, + min_disk, + min_ram, + protected, + virtual_size + ) + SELECT id, + name, + size, + status, + created_at, + updated_at, + deleted_at, + deleted, + disk_format, + container_format, + checksum, + owner, + min_disk, + min_ram, + protected, + virtual_size + FROM images_backup + WHERE is_public=0; + +UPDATE images SET visibility='private' WHERE visibility='shared'; +UPDATE images SET visibility='shared' WHERE visibility='private' AND id IN (SELECT DISTINCT image_id FROM image_members WHERE deleted != 1); + +DROP TABLE images_backup; diff --git a/glance/tests/unit/test_manage.py b/glance/tests/unit/test_manage.py index 16a76aba..0211e10e 100644 --- a/glance/tests/unit/test_manage.py +++ b/glance/tests/unit/test_manage.py @@ -15,11 +15,8 @@ import fixtures import mock -from oslo_db.sqlalchemy import migration -from six.moves import StringIO from glance.cmd import manage -from glance.db import migration as db_migration from glance.db.sqlalchemy import api as db_api from glance.db.sqlalchemy import metadata as db_metadata from glance.tests import utils as test_utils @@ -51,48 +48,35 @@ class TestManageBase(test_utils.BaseTestCase): class TestLegacyManage(TestManageBase): - @mock.patch.object(migration, 'db_version') - def test_legacy_db_version(self, db_version): - with mock.patch('sys.stdout', new_callable=StringIO): - self._main_test_helper(['glance.cmd.manage', 'db_version'], - migration.db_version, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, 0) + @mock.patch.object(manage.DbCommands, 'version') + def test_legacy_db_version(self, db_upgrade): + self._main_test_helper(['glance.cmd.manage', 'db_version'], + manage.DbCommands.version) - @mock.patch.object(migration, 'db_sync') + @mock.patch.object(manage.DbCommands, 'sync') def test_legacy_db_sync(self, db_sync): self._main_test_helper(['glance.cmd.manage', 'db_sync'], - migration.db_sync, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, None) + manage.DbCommands.sync, None) - @mock.patch.object(migration, 'db_sync') - def test_legacy_db_upgrade(self, db_sync): + @mock.patch.object(manage.DbCommands, 'upgrade') + def test_legacy_db_upgrade(self, db_upgrade): self._main_test_helper(['glance.cmd.manage', 'db_upgrade'], - migration.db_sync, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, None) + manage.DbCommands.upgrade, None) - @mock.patch.object(migration, 'db_version_control') + @mock.patch.object(manage.DbCommands, 'version_control') def test_legacy_db_version_control(self, db_version_control): self._main_test_helper(['glance.cmd.manage', 'db_version_control'], - migration.db_version_control, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, None) + manage.DbCommands.version_control, None) - @mock.patch.object(migration, 'db_sync') + @mock.patch.object(manage.DbCommands, 'sync') def test_legacy_db_sync_version(self, db_sync): - self._main_test_helper(['glance.cmd.manage', 'db_sync', '20'], - migration.db_sync, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, '20') + self._main_test_helper(['glance.cmd.manage', 'db_sync', 'liberty'], + manage.DbCommands.sync, 'liberty') - @mock.patch.object(migration, 'db_sync') - def test_legacy_db_upgrade_version(self, db_sync): - self._main_test_helper(['glance.cmd.manage', 'db_upgrade', '20'], - migration.db_sync, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, '20') + @mock.patch.object(manage.DbCommands, 'upgrade') + def test_legacy_db_upgrade_version(self, db_upgrade): + self._main_test_helper(['glance.cmd.manage', 'db_upgrade', 'liberty'], + manage.DbCommands.upgrade, 'liberty') def test_db_metadefs_unload(self): db_metadata.db_unload_metadefs = mock.Mock() @@ -157,48 +141,36 @@ class TestLegacyManage(TestManageBase): class TestManage(TestManageBase): - @mock.patch.object(migration, 'db_version') - def test_db_version(self, db_version): - with mock.patch('sys.stdout', new_callable=StringIO): - self._main_test_helper(['glance.cmd.manage', 'db', 'version'], - migration.db_version, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, 0) + @mock.patch.object(manage.DbCommands, 'version') + def test_db_version(self, version): + self._main_test_helper(['glance.cmd.manage', 'db', 'version'], + manage.DbCommands.version) - @mock.patch.object(migration, 'db_sync') - def test_db_sync(self, db_sync): + @mock.patch.object(manage.DbCommands, 'sync') + def test_db_sync(self, sync): self._main_test_helper(['glance.cmd.manage', 'db', 'sync'], - migration.db_sync, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, None) + manage.DbCommands.sync) - @mock.patch.object(migration, 'db_sync') - def test_db_upgrade(self, db_sync): + @mock.patch.object(manage.DbCommands, 'upgrade') + def test_db_upgrade(self, upgrade): self._main_test_helper(['glance.cmd.manage', 'db', 'upgrade'], - migration.db_sync, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, None) + manage.DbCommands.upgrade) - @mock.patch.object(migration, 'db_version_control') - def test_db_version_control(self, db_version_control): + @mock.patch.object(manage.DbCommands, 'version_control') + def test_db_version_control(self, version_control): self._main_test_helper(['glance.cmd.manage', 'db', 'version_control'], - migration.db_version_control, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, None) + manage.DbCommands.version_control) - @mock.patch.object(migration, 'db_sync') - def test_db_sync_version(self, db_sync): - self._main_test_helper(['glance.cmd.manage', 'db', 'sync', '20'], - migration.db_sync, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, '20') + @mock.patch.object(manage.DbCommands, 'sync') + def test_db_sync_version(self, sync): + self._main_test_helper(['glance.cmd.manage', 'db', 'sync', 'liberty'], + manage.DbCommands.sync, 'liberty') - @mock.patch.object(migration, 'db_sync') - def test_db_upgrade_version(self, db_sync): - self._main_test_helper(['glance.cmd.manage', 'db', 'upgrade', '20'], - migration.db_sync, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, '20') + @mock.patch.object(manage.DbCommands, 'upgrade') + def test_db_upgrade_version(self, upgrade): + self._main_test_helper(['glance.cmd.manage', 'db', + 'upgrade', 'liberty'], + manage.DbCommands.upgrade, 'liberty') def test_db_metadefs_unload(self): db_metadata.db_unload_metadefs = mock.Mock() diff --git a/requirements.txt b/requirements.txt index ff3ef378..16369c29 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,8 @@ Routes!=2.0,!=2.1,!=2.3.0,>=1.12.3;python_version=='2.7' # MIT Routes!=2.0,!=2.3.0,>=1.12.3;python_version!='2.7' # MIT WebOb>=1.6.0 # MIT sqlalchemy-migrate>=0.9.6 # Apache-2.0 +sqlparse>=0.2.2 # BSD +alembic>=0.8.10 # MIT httplib2>=0.7.5 # MIT pycrypto>=2.6 # Public Domain oslo.config!=3.18.0,>=3.14.0 # Apache-2.0