diff --git a/glance/cmd/manage.py b/glance/cmd/manage.py index 0daac4e5ef..79e08f9b00 100644 --- a/glance/cmd/manage.py +++ b/glance/cmd/manage.py @@ -90,9 +90,9 @@ class DbCommands(object): 'alembic migration control.')) @args('--version', metavar='', help='Database version') - def upgrade(self, version=db_migration.LATEST_REVISION): + def upgrade(self, version='heads'): """Upgrade the database's migration level""" - self.sync(version) + self._sync(version) @args('--version', metavar='', help='Database version') def version_control(self, version=db_migration.ALEMBIC_INIT_VERSION): @@ -107,12 +107,26 @@ class DbCommands(object): "revision:"), version) @args('--version', metavar='', help='Database version') - def sync(self, version=db_migration.LATEST_REVISION): + def sync(self, version=None): + curr_heads = alembic_migrations.get_current_alembic_heads() + contract = alembic_migrations.get_alembic_branch_head( + db_migration.CONTRACT_BRANCH) + + if (contract in curr_heads): + sys.exit(_('Database is up to date. No migrations needed.')) + + try: + self.expand() + self.migrate() + self.contract() + print(_('Database is synced successfully.')) + except exception.GlanceException as e: + sys.exit(_('Failed to sync database: ERROR: %s') % e) + + def _sync(self, version): """ Place an existing database under migration control and upgrade it. """ - if version is None: - version = db_migration.LATEST_REVISION alembic_migrations.place_database_under_alembic_control() @@ -128,20 +142,35 @@ class DbCommands(object): print(_('Upgraded database to: %(v)s, current revision(s): %(r)s') % {'v': version, 'r': revs}) + def _validate_engine(self, engine): + """Check engine is valid or not. + + MySql is only supported for Glance EMC. + Adding sqlite as engine to support existing functional test cases. + + :param engine: database engine name + """ + if engine.engine.name not in ['mysql', 'sqlite']: + sys.exit(_('Rolling upgrades are currently supported only for ' + 'MySQL and Sqlite')) + def expand(self): """Run the expansion phase of a rolling upgrade procedure.""" - engine = db_api.get_engine() - if engine.engine.name != 'mysql': - sys.exit(_('Rolling upgrades are currently supported only for ' - 'MySQL')) + self._validate_engine(db_api.get_engine()) + curr_heads = alembic_migrations.get_current_alembic_heads() expand_head = alembic_migrations.get_alembic_branch_head( db_migration.EXPAND_BRANCH) + contract_head = alembic_migrations.get_alembic_branch_head( + db_migration.CONTRACT_BRANCH) + if not expand_head: sys.exit(_('Database expansion failed. Couldn\'t find head ' 'revision of expand branch.')) + elif (contract_head in curr_heads): + sys.exit(_('Database is up to date. No migrations needed.')) - self.sync(version=expand_head) + self._sync(version=expand_head) curr_heads = alembic_migrations.get_current_alembic_heads() if expand_head not in curr_heads: @@ -152,18 +181,18 @@ class DbCommands(object): def contract(self): """Run the contraction phase of a rolling upgrade procedure.""" - engine = db_api.get_engine() - if engine.engine.name != 'mysql': - sys.exit(_('Rolling upgrades are currently supported only for ' - 'MySQL')) + self._validate_engine(db_api.get_engine()) + curr_heads = alembic_migrations.get_current_alembic_heads() contract_head = alembic_migrations.get_alembic_branch_head( db_migration.CONTRACT_BRANCH) + if not contract_head: sys.exit(_('Database contraction failed. Couldn\'t find head ' 'revision of contract branch.')) + elif (contract_head in curr_heads): + sys.exit(_('Database is up to date. No migrations needed.')) - curr_heads = alembic_migrations.get_current_alembic_heads() expand_head = alembic_migrations.get_alembic_branch_head( db_migration.EXPAND_BRANCH) if expand_head not in curr_heads: @@ -178,7 +207,7 @@ class DbCommands(object): 'complete. Run data migration using "glance-manage db ' 'migrate".')) - self.sync(version=contract_head) + self._sync(version=contract_head) curr_heads = alembic_migrations.get_current_alembic_heads() if contract_head not in curr_heads: @@ -189,12 +218,15 @@ class DbCommands(object): 'curr_revs': curr_heads}) def migrate(self): - engine = db_api.get_engine() - if engine.engine.name != 'mysql': - sys.exit(_('Rolling upgrades are currently supported only for ' - 'MySQL')) + self._validate_engine(db_api.get_engine()) curr_heads = alembic_migrations.get_current_alembic_heads() + contract_head = alembic_migrations.get_alembic_branch_head( + db_migration.CONTRACT_BRANCH) + + if (contract_head in curr_heads): + sys.exit(_('Database is up to date. No migrations needed.')) + expand_head = alembic_migrations.get_alembic_branch_head( db_migration.EXPAND_BRANCH) if expand_head not in curr_heads: @@ -281,13 +313,13 @@ class DbLegacyCommands(object): def version(self): self.command_object.version() - def upgrade(self, version=db_migration.LATEST_REVISION): + def upgrade(self, version='heads'): self.command_object.upgrade(CONF.command.version) def version_control(self, version=db_migration.ALEMBIC_INIT_VERSION): self.command_object.version_control(CONF.command.version) - def sync(self, version=db_migration.LATEST_REVISION): + def sync(self, version=None): self.command_object.sync(CONF.command.version) def expand(self): diff --git a/glance/db/migration.py b/glance/db/migration.py index bf0e7cd9b6..b941114bfc 100644 --- a/glance/db/migration.py +++ b/glance/db/migration.py @@ -49,7 +49,7 @@ EXPAND_BRANCH = 'expand' CONTRACT_BRANCH = 'contract' CURRENT_RELEASE = 'queens' ALEMBIC_INIT_VERSION = 'liberty' -LATEST_REVISION = 'pike01' +LATEST_REVISION = 'queens_contract01' INIT_VERSION = 0 MIGRATE_REPO_PATH = os.path.join( diff --git a/glance/db/sqlalchemy/alembic_migrations/__init__.py b/glance/db/sqlalchemy/alembic_migrations/__init__.py index 57534f6946..f35e47d0a0 100644 --- a/glance/db/sqlalchemy/alembic_migrations/__init__.py +++ b/glance/db/sqlalchemy/alembic_migrations/__init__.py @@ -20,6 +20,7 @@ from alembic import command as alembic_command from alembic import config as alembic_config from alembic import migration as alembic_migration from alembic import script as alembic_script +from sqlalchemy import MetaData, Table from oslo_db import exception as db_exception from oslo_db.sqlalchemy import migration as sqla_migration @@ -44,6 +45,25 @@ def get_current_alembic_heads(): with engine.connect() as conn: context = alembic_migration.MigrationContext.configure(conn) heads = context.get_current_heads() + + def update_alembic_version(old, new): + """Correct alembic head in order to upgrade DB using EMC method. + + :param:old: Actual alembic head + :param:new: Expected alembic head to be updated + """ + meta = MetaData(engine) + alembic_version = Table('alembic_version', meta, autoload=True) + alembic_version.update().values( + version_num=new).where( + alembic_version.c.version_num == old).execute() + + if ("pike01" in heads): + update_alembic_version("pike01", "pike_contract01") + elif ("ocata01" in heads): + update_alembic_version("ocata01", "ocata_contract01") + + heads = context.get_current_heads() return heads diff --git a/glance/db/sqlalchemy/alembic_migrations/data_migrations/__init__.py b/glance/db/sqlalchemy/alembic_migrations/data_migrations/__init__.py index 4d65ff1cff..e8636ac2d3 100644 --- a/glance/db/sqlalchemy/alembic_migrations/data_migrations/__init__.py +++ b/glance/db/sqlalchemy/alembic_migrations/data_migrations/__init__.py @@ -51,20 +51,20 @@ def _run_migrations(engine, migrations): return rows_migrated -def has_pending_migrations(engine=None): +def has_pending_migrations(engine=None, release=db_migrations.CURRENT_RELEASE): if not engine: engine = db_api.get_engine() - migrations = _find_migration_modules(db_migrations.CURRENT_RELEASE) + migrations = _find_migration_modules(release) if not migrations: return False return any([x.has_migrations(engine) for x in migrations]) -def migrate(engine=None): +def migrate(engine=None, release=db_migrations.CURRENT_RELEASE): if not engine: engine = db_api.get_engine() - migrations = _find_migration_modules(db_migrations.CURRENT_RELEASE) + migrations = _find_migration_modules(release) rows_migrated = _run_migrations(engine, migrations) return rows_migrated diff --git a/glance/db/sqlalchemy/alembic_migrations/data_migrations/queens_migrate01_empty.py b/glance/db/sqlalchemy/alembic_migrations/data_migrations/queens_migrate01_empty.py new file mode 100644 index 0000000000..6a56183cf9 --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/data_migrations/queens_migrate01_empty.py @@ -0,0 +1,26 @@ +# Copyright (C) 2018 NTT DATA +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def has_migrations(engine): + """Returns true if at least one data row can be migrated.""" + + return False + + +def migrate(engine): + """Return the number of rows migrated.""" + + return 0 diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.py b/glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.py deleted file mode 100644 index 5d66513e36..0000000000 --- a/glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.py +++ /dev/null @@ -1,72 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""add visibility to and remove is_public from images - -Revision ID: ocata01 -Revises: mitaka02 -Create Date: 2017-01-20 12:58:16.647499 - -""" - -import os - -from alembic import op -from sqlalchemy import Column, Enum, MetaData, select, Table, not_, and_ -import sqlparse - -# revision identifiers, used by Alembic. -revision = 'ocata01' -down_revision = 'mitaka02' -branch_labels = None -depends_on = None - - -def upgrade(): - migrate_engine = op.get_bind() - meta = MetaData(bind=migrate_engine) - - engine_name = migrate_engine.engine.name - if engine_name == 'sqlite': - sql_file = os.path.splitext(__file__)[0] - sql_file += '.sql' - with open(sql_file, 'r') as sqlite_script: - sql = sqlparse.format(sqlite_script.read(), strip_comments=True) - for statement in sqlparse.split(sql): - op.execute(statement) - return - - enum = Enum('private', 'public', 'shared', 'community', metadata=meta, - name='image_visibility') - enum.create() - v_col = Column('visibility', enum, nullable=False, server_default='shared') - op.add_column('images', v_col) - - op.create_index('visibility_image_idx', 'images', ['visibility']) - - images = Table('images', meta, autoload=True) - images.update(values={'visibility': 'public'}).where( - images.c.is_public).execute() - - image_members = Table('image_members', meta, autoload=True) - - # NOTE(dharinic): Mark all the non-public images as 'private' first - images.update().values(visibility='private').where( - not_(images.c.is_public)).execute() - # NOTE(dharinic): Identify 'shared' images from the above - images.update().values(visibility='shared').where(and_( - images.c.visibility == 'private', images.c.id.in_(select( - [image_members.c.image_id]).distinct().where( - not_(image_members.c.deleted))))).execute() - - op.drop_index('ix_images_is_public', 'images') - op.drop_column('images', 'is_public') diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.sql b/glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.sql deleted file mode 100644 index 0e848ccefb..0000000000 --- a/glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.sql +++ /dev/null @@ -1,162 +0,0 @@ -CREATE TEMPORARY TABLE images_backup ( - id VARCHAR(36) NOT NULL, - name VARCHAR(255), - size INTEGER, - status VARCHAR(30) NOT NULL, - is_public BOOLEAN NOT NULL, - created_at DATETIME NOT NULL, - updated_at DATETIME, - deleted_at DATETIME, - deleted BOOLEAN NOT NULL, - disk_format VARCHAR(20), - container_format VARCHAR(20), - checksum VARCHAR(32), - owner VARCHAR(255), - min_disk INTEGER NOT NULL, - min_ram INTEGER NOT NULL, - protected BOOLEAN DEFAULT 0 NOT NULL, - virtual_size INTEGER, - PRIMARY KEY (id), - CHECK (is_public IN (0, 1)), - CHECK (deleted IN (0, 1)), - CHECK (protected IN (0, 1)) -); - -INSERT INTO images_backup - SELECT id, - name, - size, - status, - is_public, - created_at, - updated_at, - deleted_at, - deleted, - disk_format, - container_format, - checksum, - owner, - min_disk, - min_ram, - protected, - virtual_size - FROM images; - -DROP TABLE images; - -CREATE TABLE images ( - id VARCHAR(36) NOT NULL, - name VARCHAR(255), - size INTEGER, - status VARCHAR(30) NOT NULL, - created_at DATETIME NOT NULL, - updated_at DATETIME, - deleted_at DATETIME, - deleted BOOLEAN NOT NULL, - disk_format VARCHAR(20), - container_format VARCHAR(20), - checksum VARCHAR(32), - owner VARCHAR(255), - min_disk INTEGER NOT NULL, - min_ram INTEGER NOT NULL, - protected BOOLEAN DEFAULT 0 NOT NULL, - virtual_size INTEGER, - visibility VARCHAR(9) DEFAULT 'shared' NOT NULL, - PRIMARY KEY (id), - CHECK (deleted IN (0, 1)), - CHECK (protected IN (0, 1)), - CONSTRAINT image_visibility CHECK (visibility IN ('private', 'public', 'shared', 'community')) -); - -CREATE INDEX checksum_image_idx ON images (checksum); -CREATE INDEX visibility_image_idx ON images (visibility); -CREATE INDEX ix_images_deleted ON images (deleted); -CREATE INDEX owner_image_idx ON images (owner); -CREATE INDEX created_at_image_idx ON images (created_at); -CREATE INDEX updated_at_image_idx ON images (updated_at); - --- Copy over all the 'public' rows - -INSERT INTO images ( - id, - name, - size, - status, - created_at, - updated_at, - deleted_at, - deleted, - disk_format, - container_format, - checksum, - owner, - min_disk, - min_ram, - protected, - virtual_size - ) - SELECT id, - name, - size, - status, - created_at, - updated_at, - deleted_at, - deleted, - disk_format, - container_format, - checksum, - owner, - min_disk, - min_ram, - protected, - virtual_size - FROM images_backup - WHERE is_public=1; - - -UPDATE images SET visibility='public'; - --- Now copy over the 'private' rows - -INSERT INTO images ( - id, - name, - size, - status, - created_at, - updated_at, - deleted_at, - deleted, - disk_format, - container_format, - checksum, - owner, - min_disk, - min_ram, - protected, - virtual_size - ) - SELECT id, - name, - size, - status, - created_at, - updated_at, - deleted_at, - deleted, - disk_format, - container_format, - checksum, - owner, - min_disk, - min_ram, - protected, - virtual_size - FROM images_backup - WHERE is_public=0; - -UPDATE images SET visibility='private' WHERE visibility='shared'; -UPDATE images SET visibility='shared' WHERE visibility='private' AND id IN (SELECT DISTINCT image_id FROM image_members WHERE deleted != 1); - -DROP TABLE images_backup; diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/ocata_contract01_drop_is_public.py b/glance/db/sqlalchemy/alembic_migrations/versions/ocata_contract01_drop_is_public.py index 48cb12c086..8f382bae0d 100644 --- a/glance/db/sqlalchemy/alembic_migrations/versions/ocata_contract01_drop_is_public.py +++ b/glance/db/sqlalchemy/alembic_migrations/versions/ocata_contract01_drop_is_public.py @@ -19,14 +19,14 @@ Create Date: 2017-01-27 12:58:16.647499 """ from alembic import op -from sqlalchemy import MetaData, Table +from sqlalchemy import MetaData, Enum from glance.db import migration # revision identifiers, used by Alembic. revision = 'ocata_contract01' down_revision = 'mitaka02' -branch_labels = migration.CONTRACT_BRANCH +branch_labels = ('ocata01', migration.CONTRACT_BRANCH) depends_on = 'ocata_expand01' @@ -40,8 +40,9 @@ DROP TRIGGER update_visibility; def _drop_column(): - op.drop_index('ix_images_is_public', 'images') - op.drop_column('images', 'is_public') + with op.batch_alter_table('images') as batch_op: + batch_op.drop_index('ix_images_is_public') + batch_op.drop_column('is_public') def _drop_triggers(engine): @@ -54,8 +55,14 @@ def _drop_triggers(engine): def _set_nullability_and_default_on_visibility(meta): # NOTE(hemanthm): setting the default on 'visibility' column # to 'shared'. Also, marking it as non-nullable. - images = Table('images', meta, autoload=True) - images.c.visibility.alter(nullable=False, server_default='shared') + # images = Table('images', meta, autoload=True) + existing_type = Enum('private', 'public', 'shared', 'community', + metadata=meta, name='image_visibility') + with op.batch_alter_table('images') as batch_op: + batch_op.alter_column('visibility', + nullable=False, + server_default='shared', + existing_type=existing_type) def upgrade(): diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/queens_contract01_empty.py b/glance/db/sqlalchemy/alembic_migrations/versions/queens_contract01_empty.py new file mode 100644 index 0000000000..2057bea9e1 --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/queens_contract01_empty.py @@ -0,0 +1,25 @@ +# Copyright (C) 2018 NTT DATA +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +# revision identifiers, used by Alembic. +revision = 'queens_contract01' +down_revision = 'pike_contract01' +branch_labels = None +depends_on = 'queens_expand01' + + +def upgrade(): + pass diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/pike01_drop_artifacts_tables.py b/glance/db/sqlalchemy/alembic_migrations/versions/queens_expand01_empty.py similarity index 52% rename from glance/db/sqlalchemy/alembic_migrations/versions/pike01_drop_artifacts_tables.py rename to glance/db/sqlalchemy/alembic_migrations/versions/queens_expand01_empty.py index b7886c47dc..aee4cb018c 100644 --- a/glance/db/sqlalchemy/alembic_migrations/versions/pike01_drop_artifacts_tables.py +++ b/glance/db/sqlalchemy/alembic_migrations/versions/queens_expand01_empty.py @@ -1,3 +1,6 @@ +# Copyright (C) 2018 NTT DATA +# All Rights Reserved. +# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -10,32 +13,13 @@ # License for the specific language governing permissions and limitations # under the License. -"""drop glare artifacts tables - -Revision ID: pike01 -Revises: ocata01 -Create Date: 2017-02-08 20:32:51.200867 - -""" - -from alembic import op # revision identifiers, used by Alembic. -revision = 'pike01' -down_revision = 'ocata01' +revision = 'queens_expand01' +down_revision = 'pike_expand01' branch_labels = None depends_on = None def upgrade(): - # create list of artifact tables in reverse order of their creation - table_names = [] - table_names.append('artifact_blob_locations') - table_names.append('artifact_properties') - table_names.append('artifact_blobs') - table_names.append('artifact_dependencies') - table_names.append('artifact_tags') - table_names.append('artifacts') - - for table_name in table_names: - op.drop_table(table_name=table_name) + pass diff --git a/glance/tests/functional/db/migrations/test_ocata01.py b/glance/tests/functional/db/migrations/test_ocata01.py deleted file mode 100644 index 323fee310e..0000000000 --- a/glance/tests/functional/db/migrations/test_ocata01.py +++ /dev/null @@ -1,142 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import datetime - -from oslo_db.sqlalchemy import test_base -from oslo_db.sqlalchemy import utils as db_utils - -from glance.tests.functional.db import test_migrations - - -class TestOcata01Mixin(test_migrations.AlembicMigrationsMixin): - - def _pre_upgrade_ocata01(self, engine): - images = db_utils.get_table(engine, 'images') - now = datetime.datetime.now() - image_members = db_utils.get_table(engine, 'image_members') - - # inserting a public image record - public_temp = dict(deleted=False, - created_at=now, - status='active', - is_public=True, - min_disk=0, - min_ram=0, - id='public_id') - images.insert().values(public_temp).execute() - - # inserting a non-public image record for 'shared' visibility test - shared_temp = dict(deleted=False, - created_at=now, - status='active', - is_public=False, - min_disk=0, - min_ram=0, - id='shared_id') - images.insert().values(shared_temp).execute() - - # inserting a non-public image records for 'private' visibility test - private_temp = dict(deleted=False, - created_at=now, - status='active', - is_public=False, - min_disk=0, - min_ram=0, - id='private_id_1') - images.insert().values(private_temp).execute() - private_temp = dict(deleted=False, - created_at=now, - status='active', - is_public=False, - min_disk=0, - min_ram=0, - id='private_id_2') - images.insert().values(private_temp).execute() - - # adding an active as well as a deleted image member for checking - # 'shared' visibility - temp = dict(deleted=False, - created_at=now, - image_id='shared_id', - member='fake_member_452', - can_share=True, - id=45) - image_members.insert().values(temp).execute() - - temp = dict(deleted=True, - created_at=now, - image_id='shared_id', - member='fake_member_453', - can_share=True, - id=453) - image_members.insert().values(temp).execute() - - # adding an image member, but marking it deleted, - # for testing 'private' visibility - temp = dict(deleted=True, - created_at=now, - image_id='private_id_2', - member='fake_member_451', - can_share=True, - id=451) - image_members.insert().values(temp).execute() - - # adding an active image member for the 'public' image, - # to test it remains public regardless. - temp = dict(deleted=False, - created_at=now, - image_id='public_id', - member='fake_member_450', - can_share=True, - id=450) - image_members.insert().values(temp).execute() - - def _check_ocata01(self, engine, data): - # check that after migration, 'visibility' column is introduced - images = db_utils.get_table(engine, 'images') - self.assertIn('visibility', images.c) - self.assertNotIn('is_public', images.c) - - # tests to identify the visibilities of images created above - rows = images.select().where( - images.c.id == 'public_id').execute().fetchall() - self.assertEqual(1, len(rows)) - self.assertEqual('public', rows[0][16]) - - rows = images.select().where( - images.c.id == 'shared_id').execute().fetchall() - self.assertEqual(1, len(rows)) - self.assertEqual('shared', rows[0][16]) - - rows = images.select().where( - images.c.id == 'private_id_1').execute().fetchall() - self.assertEqual(1, len(rows)) - self.assertEqual('private', rows[0][16]) - - rows = images.select().where( - images.c.id == 'private_id_2').execute().fetchall() - self.assertEqual(1, len(rows)) - self.assertEqual('private', rows[0][16]) - - -class TestOcata01MySQL(TestOcata01Mixin, test_base.MySQLOpportunisticTestCase): - pass - - -class TestOcata01PostgresSQL(TestOcata01Mixin, - test_base.PostgreSQLOpportunisticTestCase): - pass - - -class TestOcata01Sqlite(TestOcata01Mixin, test_base.DbTestCase): - pass diff --git a/glance/tests/functional/db/migrations/test_ocata_contract01.py b/glance/tests/functional/db/migrations/test_ocata_contract01.py index b2049f63ed..9400fe5a66 100644 --- a/glance/tests/functional/db/migrations/test_ocata_contract01.py +++ b/glance/tests/functional/db/migrations/test_ocata_contract01.py @@ -15,6 +15,7 @@ import datetime from oslo_db.sqlalchemy import test_base from oslo_db.sqlalchemy import utils as db_utils +from glance.db.sqlalchemy.alembic_migrations import data_migrations from glance.tests.functional.db import test_migrations @@ -52,6 +53,8 @@ class TestOcataContract01Mixin(test_migrations.AlembicMigrationsMixin): id='private_id_before_expand') images.insert().values(shared_temp).execute() + data_migrations.migrate(engine=engine, release='ocata') + def _check_ocata_contract01(self, engine, data): # check that after contract 'is_public' column is dropped images = db_utils.get_table(engine, 'images') diff --git a/glance/tests/functional/db/migrations/test_pike01.py b/glance/tests/functional/db/migrations/test_pike01.py deleted file mode 100644 index fcb6db4605..0000000000 --- a/glance/tests/functional/db/migrations/test_pike01.py +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_db.sqlalchemy import test_base -from oslo_db.sqlalchemy import utils as db_utils -import sqlalchemy - -from glance.tests.functional.db import test_migrations - - -class TestPike01Mixin(test_migrations.AlembicMigrationsMixin): - - artifacts_table_names = [ - 'artifact_blob_locations', - 'artifact_properties', - 'artifact_blobs', - 'artifact_dependencies', - 'artifact_tags', - 'artifacts' - ] - - def _pre_upgrade_pike01(self, engine): - # verify presence of the artifacts tables - for table_name in self.artifacts_table_names: - table = db_utils.get_table(engine, table_name) - self.assertIsNotNone(table) - - def _check_pike01(self, engine, data): - # verify absence of the artifacts tables - for table_name in self.artifacts_table_names: - self.assertRaises(sqlalchemy.exc.NoSuchTableError, - db_utils.get_table, engine, table_name) - - -class TestPike01MySQL(TestPike01Mixin, test_base.MySQLOpportunisticTestCase): - pass - - -class TestPike01PostgresSQL(TestPike01Mixin, - test_base.PostgreSQLOpportunisticTestCase): - pass - - -class TestPike01Sqlite(TestPike01Mixin, test_base.DbTestCase): - pass diff --git a/glance/tests/functional/db/test_migrations.py b/glance/tests/functional/db/test_migrations.py index 8bc6ea55f3..bbe226d607 100644 --- a/glance/tests/functional/db/test_migrations.py +++ b/glance/tests/functional/db/test_migrations.py @@ -23,7 +23,6 @@ from oslo_db.sqlalchemy import test_migrations from oslo_db.tests.sqlalchemy import base as test_base import sqlalchemy.types as types -from glance.db import migration as db_migration from glance.db.sqlalchemy import alembic_migrations from glance.db.sqlalchemy.alembic_migrations import versions from glance.db.sqlalchemy import models @@ -34,7 +33,7 @@ import glance.tests.utils as test_utils class AlembicMigrationsMixin(object): def _get_revisions(self, config, head=None): - head = head or db_migration.LATEST_REVISION + head = head or 'heads' scripts_dir = alembic_script.ScriptDirectory.from_config(config) revisions = list(scripts_dir.walk_revisions(base='base', head=head)) diff --git a/glance/tests/functional/test_glance_manage.py b/glance/tests/functional/test_glance_manage.py index 655f09e000..278c4a4169 100644 --- a/glance/tests/functional/test_glance_manage.py +++ b/glance/tests/functional/test_glance_manage.py @@ -18,12 +18,19 @@ import os import sys +from oslo_config import cfg +from oslo_db import options as db_options + from glance.common import utils +from glance.db import migration as db_migration +from glance.db.sqlalchemy import alembic_migrations from glance.tests import functional from glance.tests.utils import depends_on_exe from glance.tests.utils import execute from glance.tests.utils import skip_if_disabled +CONF = cfg.CONF + class TestGlanceManage(functional.FunctionalTest): """Functional tests for glance-manage""" @@ -36,6 +43,8 @@ class TestGlanceManage(functional.FunctionalTest): self.db_filepath = os.path.join(self.test_dir, 'tests.sqlite') self.connection = ('sql_connection = sqlite:///%s' % self.db_filepath) + db_options.set_defaults(CONF, connection='sqlite:///%s' % + self.db_filepath) def _sync_db(self): with open(self.conf_filepath, 'w') as conf_file: @@ -64,3 +73,16 @@ class TestGlanceManage(functional.FunctionalTest): for table in ['images', 'image_tags', 'image_locations', 'image_members', 'image_properties']: self._assert_table_exists(table) + + @depends_on_exe('sqlite3') + @skip_if_disabled + def test_sync(self): + """Test DB sync which internally calls EMC""" + self._sync_db() + contract_head = alembic_migrations.get_alembic_branch_head( + db_migration.CONTRACT_BRANCH) + + cmd = ("sqlite3 {0} \"SELECT version_num FROM alembic_version\"" + ).format(self.db_filepath) + exitcode, out, err = execute(cmd, raise_error=True) + self.assertEqual(contract_head, out.rstrip().decode("utf-8")) diff --git a/glance/tests/unit/test_data_migration_framework.py b/glance/tests/unit/test_data_migration_framework.py index e23f7be937..759f06e422 100644 --- a/glance/tests/unit/test_data_migration_framework.py +++ b/glance/tests/unit/test_data_migration_framework.py @@ -197,7 +197,7 @@ class TestDataMigrationFramework(test_utils.BaseTestCase): mock_import.side_effect = fake_imported_modules engine = mock.Mock() - actual = data_migrations.migrate(engine) + actual = data_migrations.migrate(engine, 'zebra') self.assertEqual(150, actual) zebra1.has_migrations.assert_called_once_with(engine) zebra1.migrate.assert_called_once_with(engine) diff --git a/glance/tests/utils.py b/glance/tests/utils.py index 41a455b38b..32d85c3a6c 100644 --- a/glance/tests/utils.py +++ b/glance/tests/utils.py @@ -43,7 +43,6 @@ from glance.common import timeutils from glance.common import utils from glance.common import wsgi from glance import context -from glance.db import migration as db_migration from glance.db.sqlalchemy import alembic_migrations from glance.db.sqlalchemy import api as db_api from glance.db.sqlalchemy import models as db_models @@ -682,10 +681,8 @@ class HttplibWsgiAdapter(object): response.body) -def db_sync(version=None, engine=None): +def db_sync(version='heads', engine=None): """Migrate the database to `version` or the most recent version.""" - if version is None: - version = db_migration.LATEST_REVISION if engine is None: engine = db_api.get_engine()