Merge "Add expand/migrate/contract migrations for CI"

glance/cmd/manage.py

@@ -129,6 +129,11 @@ class DbCommands(object):
 
     def expand(self):
         """Run the expansion phase of a rolling upgrade procedure."""
+        engine = db_api.get_engine()
+        if engine.engine.name != 'mysql':
+            sys.exit(_('Rolling upgrades are currently supported only for '
+                       'MySQL'))
+
         expand_head = alembic_migrations.get_alembic_branch_head(
             db_migration.EXPAND_BRANCH)
         if not expand_head:
@@ -146,6 +151,11 @@ class DbCommands(object):
 
     def contract(self):
         """Run the contraction phase of a rolling upgrade procedure."""
+        engine = db_api.get_engine()
+        if engine.engine.name != 'mysql':
+            sys.exit(_('Rolling upgrades are currently supported only for '
+                       'MySQL'))
+
         contract_head = alembic_migrations.get_alembic_branch_head(
             db_migration.CONTRACT_BRANCH)
         if not contract_head:
@@ -178,6 +188,11 @@ class DbCommands(object):
                                             'curr_revs': curr_heads})
 
     def migrate(self):
+        engine = db_api.get_engine()
+        if engine.engine.name != 'mysql':
+            sys.exit(_('Rolling upgrades are currently supported only for '
+                       'MySQL'))
+
         curr_heads = alembic_migrations.get_current_alembic_heads()
         expand_head = alembic_migrations.get_alembic_branch_head(
             db_migration.EXPAND_BRANCH)

glance/db/sqlalchemy/alembic_migrations/data_migrations/ocata_migrate01_community_images.py

@@ -0,0 +1,103 @@
+# Copyright 2016 Rackspace
+# Copyright 2016 Intel Corporation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from sqlalchemy import MetaData, select, Table, and_, not_
+
+
+def has_migrations(engine):
+    """Returns true if at least one data row can be migrated.
+
+    There are rows left to migrate if:
+     #1 There exists a row with visibility not set yet.
+        Or
+     #2 There exists a private image with active members but its visibility
+        isn't set to 'shared' yet.
+
+    Note: This method can return a false positive if data migrations
+    are running in the background as it's being called.
+    """
+    meta = MetaData(engine)
+    images = Table('images', meta, autoload=True)
+
+    rows_with_null_visibility = (select([images.c.id])
+                                 .where(images.c.visibility == None)
+                                 .limit(1)
+                                 .execute())
+
+    if rows_with_null_visibility.rowcount == 1:
+        return True
+
+    image_members = Table('image_members', meta, autoload=True)
+    rows_with_pending_shared = (select[images.c.id]
+                                .where(and_(
+                                    images.c.visibility == 'private',
+                                    images.c.id.in_(
+                                        select([image_members.c.image_id])
+                                        .distinct()
+                                        .where(not_(image_members.c.deleted))))
+                                       )
+                                .limit(1)
+                                .execute())
+    if rows_with_pending_shared.rowcount == 1:
+        return True
+
+    return False
+
+
+def _mark_all_public_images_with_public_visibility(images):
+    migrated_rows = (images
+                     .update().values(visibility='public')
+                     .where(images.c.is_public)
+                     .execute())
+    return migrated_rows.rowcount
+
+
+def _mark_all_non_public_images_with_private_visibility(images):
+    migrated_rows = (images
+                     .update().values(visibility='private')
+                     .where(not_(images.c.is_public))
+                     .execute())
+    return migrated_rows.rowcount
+
+
+def _mark_all_private_images_with_members_as_shared_visibility(images,
+                                                               image_members):
+    migrated_rows = (images
+                     .update().values(visibility='shared')
+                     .where(and_(images.c.visibility == 'private',
+                                 images.c.id.in_(
+                                     select([image_members.c.image_id])
+                                     .distinct()
+                                     .where(not_(image_members.c.deleted)))))
+                     .execute())
+    return migrated_rows.rowcount
+
+
+def _migrate_all(engine):
+    meta = MetaData(engine)
+    images = Table('images', meta, autoload=True)
+    image_members = Table('image_members', meta, autoload=True)
+
+    num_rows = _mark_all_public_images_with_public_visibility(images)
+    num_rows += _mark_all_non_public_images_with_private_visibility(images)
+    num_rows += _mark_all_private_images_with_members_as_shared_visibility(
+        images, image_members)
+
+    return num_rows
+
+
+def migrate(engine):
+    """Set visibility column based on is_public and image members."""
+    return _migrate_all(engine)

glance/db/sqlalchemy/alembic_migrations/versions/ocata_contract01_drop_is_public.py

@@ -0,0 +1,67 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""remove is_public from images
+
+Revision ID: ocata_contract01
+Revises: mitaka02
+Create Date: 2017-01-27 12:58:16.647499
+
+"""
+
+from alembic import op
+from sqlalchemy import MetaData, Table
+
+from glance.db import migration
+
+# revision identifiers, used by Alembic.
+revision = 'ocata_contract01'
+down_revision = 'mitaka02'
+branch_labels = migration.CONTRACT_BRANCH
+depends_on = 'expand'
+
+
+MYSQL_DROP_INSERT_TRIGGER = """
+DROP TRIGGER insert_visibility;
+"""
+
+MYSQL_DROP_UPDATE_TRIGGER = """
+DROP TRIGGER update_visibility;
+"""
+
+
+def _drop_column():
+    op.drop_index('ix_images_is_public', 'images')
+    op.drop_column('images', 'is_public')
+
+
+def _drop_triggers(engine):
+    engine_name = engine.engine.name
+    if engine_name == "mysql":
+        op.execute(MYSQL_DROP_INSERT_TRIGGER)
+        op.execute(MYSQL_DROP_UPDATE_TRIGGER)
+
+
+def _set_nullability_and_default_on_visibility(meta):
+    # NOTE(hemanthm): setting the default on 'visibility' column
+    # to 'shared'. Also, marking it as non-nullable.
+    images = Table('images', meta, autoload=True)
+    images.c.visibility.alter(nullable=False, server_default='shared')
+
+
+def upgrade():
+    migrate_engine = op.get_bind()
+    meta = MetaData(bind=migrate_engine)
+
+    _drop_column()
+    _drop_triggers(migrate_engine)
+    _set_nullability_and_default_on_visibility(meta)

glance/db/sqlalchemy/alembic_migrations/versions/ocata_expand01_add_visibility.py

@@ -0,0 +1,151 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""add visibility to images
+
+Revision ID: ocata_expand01
+Revises: mitaka02
+Create Date: 2017-01-27 12:58:16.647499
+
+"""
+
+from alembic import op
+from sqlalchemy import Column, Enum, MetaData, Table
+
+from glance.db import migration
+
+# revision identifiers, used by Alembic.
+revision = 'ocata_expand01'
+down_revision = 'mitaka02'
+branch_labels = migration.EXPAND_BRANCH
+depends_on = None
+
+ERROR_MESSAGE = 'Invalid visibility value'
+MYSQL_INSERT_TRIGGER = """
+CREATE TRIGGER insert_visibility BEFORE INSERT ON images
+FOR EACH ROW
+BEGIN
+    -- NOTE(abashmak):
+    -- The following IF/ELSE block implements a priority decision tree.
+    -- Strict order MUST be followed to correctly cover all the edge cases.
+
+    -- Edge case: neither is_public nor visibility specified
+    --            (or both specified as NULL):
+    IF NEW.is_public <=> NULL AND NEW.visibility <=> NULL THEN
+        SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
+    -- Edge case: both is_public and visibility specified:
+    ELSEIF NOT(NEW.is_public <=> NULL OR NEW.visibility <=> NULL) THEN
+        SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
+    -- Inserting with is_public, set visibility accordingly:
+    ELSEIF NOT NEW.is_public <=> NULL THEN
+        IF NEW.is_public = 1 THEN
+            SET NEW.visibility = 'public';
+        ELSE
+            SET NEW.visibility = 'shared';
+        END IF;
+    -- Inserting with visibility, set is_public accordingly:
+    ELSEIF NOT NEW.visibility <=> NULL THEN
+        IF NEW.visibility = 'public' THEN
+            SET NEW.is_public = 1;
+        ELSE
+            SET NEW.is_public = 0;
+        END IF;
+    -- Edge case: either one of: is_public or visibility,
+    --            is explicitly set to NULL:
+    ELSE
+        SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
+    END IF;
+END;
+"""
+
+MYSQL_UPDATE_TRIGGER = """
+CREATE TRIGGER update_visibility BEFORE UPDATE ON images
+FOR EACH ROW
+BEGIN
+    -- Case: new value specified for is_public:
+    IF NOT NEW.is_public <=> OLD.is_public THEN
+        -- Edge case: is_public explicitly set to NULL:
+        IF NEW.is_public <=> NULL THEN
+            SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
+        -- Edge case: new value also specified for visibility
+        ELSEIF NOT NEW.visibility <=> OLD.visibility THEN
+            SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
+        -- Case: visibility not specified or specified as OLD value:
+        -- NOTE(abashmak): There is no way to reliably determine which
+        -- of the above two cases occurred, but allowing to proceed with
+        -- the update in either case does not break the model for both
+        -- N and N-1 services.
+        ELSE
+            -- Set visibility according to the value of is_public:
+            IF NEW.is_public <=> 1 THEN
+                SET NEW.visibility = 'public';
+            ELSE
+                SET NEW.visibility = 'shared';
+            END IF;
+        END IF;
+    -- Case: new value specified for visibility:
+    ELSEIF NOT NEW.visibility <=> OLD.visibility THEN
+        -- Edge case: visibility explicitly set to NULL:
+        IF NEW.visibility <=> NULL THEN
+            SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
+        -- Edge case: new value also specified for is_public
+        ELSEIF NOT NEW.is_public <=> OLD.is_public THEN
+            SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
+        -- Case: is_public not specified or specified as OLD value:
+        -- NOTE(abashmak): There is no way to reliably determine which
+        -- of the above two cases occurred, but allowing to proceed with
+        -- the update in either case does not break the model for both
+        -- N and N-1 services.
+        ELSE
+            -- Set is_public according to the value of visibility:
+            IF NEW.visibility <=> 'public' THEN
+                SET NEW.is_public = 1;
+            ELSE
+                SET NEW.is_public = 0;
+            END IF;
+        END IF;
+    END IF;
+END;
+"""
+
+
+def _add_visibility_column(meta):
+    enum = Enum('private', 'public', 'shared', 'community', metadata=meta,
+                name='image_visibility')
+    enum.create()
+    v_col = Column('visibility', enum, nullable=True, server_default=None)
+    op.add_column('images', v_col)
+    op.create_index('visibility_image_idx', 'images', ['visibility'])
+
+
+def _add_triggers(engine):
+    if engine.engine.name == 'mysql':
+        op.execute(MYSQL_INSERT_TRIGGER % (ERROR_MESSAGE, ERROR_MESSAGE,
+                                           ERROR_MESSAGE))
+        op.execute(MYSQL_UPDATE_TRIGGER % (ERROR_MESSAGE, ERROR_MESSAGE,
+                                           ERROR_MESSAGE, ERROR_MESSAGE))
+
+
+def _change_nullability_and_default_on_is_public(meta):
+    # NOTE(hemanthm): we mark is_public as nullable so that when new versions
+    # add data only to be visibility column, is_public can be null.
+    images = Table('images', meta, autoload=True)
+    images.c.is_public.alter(nullable=True, server_default=None)
+
+
+def upgrade():
+    migrate_engine = op.get_bind()
+    meta = MetaData(bind=migrate_engine)
+
+    _add_visibility_column(meta)
+    _change_nullability_and_default_on_is_public(meta)
+    _add_triggers(migrate_engine)

glance/tests/functional/db/migrations/test_ocata_contract01.py

@@ -0,0 +1,64 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import datetime
+
+from oslo_db.sqlalchemy import test_base
+from oslo_db.sqlalchemy import utils as db_utils
+
+from glance.tests.functional.db import test_migrations
+
+
+class TestOcataContract01Mixin(test_migrations.AlembicMigrationsMixin):
+
+    def _get_revisions(self, config):
+        return test_migrations.AlembicMigrationsMixin._get_revisions(
+            self, config, head='ocata_contract01')
+
+    def _pre_upgrade_ocata_contract01(self, engine):
+        images = db_utils.get_table(engine, 'images')
+        now = datetime.datetime.now()
+        self.assertIn('is_public', images.c)
+        self.assertIn('visibility', images.c)
+        self.assertTrue(images.c.is_public.nullable)
+        self.assertTrue(images.c.visibility.nullable)
+
+        # inserting a public image record
+        public_temp = dict(deleted=False,
+                           created_at=now,
+                           status='active',
+                           is_public=True,
+                           min_disk=0,
+                           min_ram=0,
+                           id='public_id_before_expand')
+        images.insert().values(public_temp).execute()
+
+        # inserting a private image record
+        shared_temp = dict(deleted=False,
+                           created_at=now,
+                           status='active',
+                           is_public=False,
+                           min_disk=0,
+                           min_ram=0,
+                           id='private_id_before_expand')
+        images.insert().values(shared_temp).execute()
+
+    def _check_ocata_contract01(self, engine, data):
+        # check that after contract 'is_public' column is dropped
+        images = db_utils.get_table(engine, 'images')
+        self.assertNotIn('is_public', images.c)
+        self.assertIn('visibility', images.c)
+
+
+class TestOcataContract01MySQL(TestOcataContract01Mixin,
+                               test_base.MySQLOpportunisticTestCase):
+    pass

glance/tests/functional/db/migrations/test_ocata_expand01.py

@@ -0,0 +1,174 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import datetime
+
+from oslo_db.sqlalchemy import test_base
+from oslo_db.sqlalchemy import utils as db_utils
+
+from glance.tests.functional.db import test_migrations
+
+
+class TestOcataExpand01Mixin(test_migrations.AlembicMigrationsMixin):
+
+    def _get_revisions(self, config):
+        return test_migrations.AlembicMigrationsMixin._get_revisions(
+            self, config, head='ocata_expand01')
+
+    def _pre_upgrade_ocata_expand01(self, engine):
+        images = db_utils.get_table(engine, 'images')
+        now = datetime.datetime.now()
+        self.assertIn('is_public', images.c)
+        self.assertNotIn('visibility', images.c)
+        self.assertFalse(images.c.is_public.nullable)
+
+        # inserting a public image record
+        public_temp = dict(deleted=False,
+                           created_at=now,
+                           status='active',
+                           is_public=True,
+                           min_disk=0,
+                           min_ram=0,
+                           id='public_id_before_expand')
+        images.insert().values(public_temp).execute()
+
+        # inserting a private image record
+        shared_temp = dict(deleted=False,
+                           created_at=now,
+                           status='active',
+                           is_public=False,
+                           min_disk=0,
+                           min_ram=0,
+                           id='private_id_before_expand')
+        images.insert().values(shared_temp).execute()
+
+    def _check_ocata_expand01(self, engine, data):
+        # check that after migration, 'visibility' column is introduced
+        images = db_utils.get_table(engine, 'images')
+        self.assertIn('visibility', images.c)
+        self.assertIn('is_public', images.c)
+        self.assertTrue(images.c.is_public.nullable)
+        self.assertTrue(images.c.visibility.nullable)
+
+        # tests visibility set to None for existing images
+        rows = (images.select()
+                .where(images.c.id.like('%_before_expand'))
+                .order_by(images.c.id)
+                .execute()
+                .fetchall())
+
+        self.assertEqual(2, len(rows))
+        # private image first
+        self.assertEqual(0, rows[0]['is_public'])
+        self.assertEqual('private_id_before_expand', rows[0]['id'])
+        self.assertIsNone(rows[0]['visibility'])
+        # then public image
+        self.assertEqual(1, rows[1]['is_public'])
+        self.assertEqual('public_id_before_expand', rows[1]['id'])
+        self.assertIsNone(rows[1]['visibility'])
+
+        self._test_trigger_old_to_new(images)
+        self._test_trigger_new_to_old(images)
+
+    def _test_trigger_new_to_old(self, images):
+        now = datetime.datetime.now()
+        # inserting a public image record after expand
+        public_temp = dict(deleted=False,
+                           created_at=now,
+                           status='active',
+                           visibility='public',
+                           min_disk=0,
+                           min_ram=0,
+                           id='public_id_new_to_old')
+        images.insert().values(public_temp).execute()
+
+        # inserting a private image record after expand
+        shared_temp = dict(deleted=False,
+                           created_at=now,
+                           status='active',
+                           visibility='private',
+                           min_disk=0,
+                           min_ram=0,
+                           id='private_id_new_to_old')
+        images.insert().values(shared_temp).execute()
+
+        # inserting a shared image record after expand
+        shared_temp = dict(deleted=False,
+                           created_at=now,
+                           status='active',
+                           visibility='shared',
+                           min_disk=0,
+                           min_ram=0,
+                           id='shared_id_new_to_old')
+        images.insert().values(shared_temp).execute()
+
+        # test visibility is set appropriately by the trigger for new images
+        rows = (images.select()
+                .where(images.c.id.like('%_new_to_old'))
+                .order_by(images.c.id)
+                .execute()
+                .fetchall())
+
+        self.assertEqual(3, len(rows))
+        # private image first
+        self.assertEqual(0, rows[0]['is_public'])
+        self.assertEqual('private_id_new_to_old', rows[0]['id'])
+        self.assertEqual('private', rows[0]['visibility'])
+        # then public image
+        self.assertEqual(1, rows[1]['is_public'])
+        self.assertEqual('public_id_new_to_old', rows[1]['id'])
+        self.assertEqual('public', rows[1]['visibility'])
+        # then shared image
+        self.assertEqual(0, rows[2]['is_public'])
+        self.assertEqual('shared_id_new_to_old', rows[2]['id'])
+        self.assertEqual('shared', rows[2]['visibility'])
+
+    def _test_trigger_old_to_new(self, images):
+        now = datetime.datetime.now()
+        # inserting a public image record after expand
+        public_temp = dict(deleted=False,
+                           created_at=now,
+                           status='active',
+                           is_public=True,
+                           min_disk=0,
+                           min_ram=0,
+                           id='public_id_old_to_new')
+        images.insert().values(public_temp).execute()
+        # inserting a private image record after expand
+        shared_temp = dict(deleted=False,
+                           created_at=now,
+                           status='active',
+                           is_public=False,
+                           min_disk=0,
+                           min_ram=0,
+                           id='private_id_old_to_new')
+        images.insert().values(shared_temp).execute()
+        # tests visibility is set appropriately by the trigger for new images
+        rows = (images.select()
+                .where(images.c.id.like('%_old_to_new'))
+                .order_by(images.c.id)
+                .execute()
+                .fetchall())
+        self.assertEqual(2, len(rows))
+        # private image first
+        self.assertEqual(0, rows[0]['is_public'])
+        self.assertEqual('private_id_old_to_new', rows[0]['id'])
+        self.assertEqual('shared', rows[0]['visibility'])
+        # then public image
+        self.assertEqual(1, rows[1]['is_public'])
+        self.assertEqual('public_id_old_to_new', rows[1]['id'])
+        self.assertEqual('public', rows[1]['visibility'])
+
+
+class TestOcataExpand01MySQL(TestOcataExpand01Mixin,
+                             test_base.MySQLOpportunisticTestCase):
+    pass

glance/tests/functional/db/migrations/test_ocata_migrate01.py

@@ -0,0 +1,147 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import datetime
+
+from oslo_db.sqlalchemy import test_base
+from oslo_db.sqlalchemy import utils as db_utils
+
+from glance.db.sqlalchemy.alembic_migrations import data_migrations
+from glance.tests.functional.db import test_migrations
+
+
+class TestOcataMigrate01Mixin(test_migrations.AlembicMigrationsMixin):
+
+    def _get_revisions(self, config):
+        return test_migrations.AlembicMigrationsMixin._get_revisions(
+            self, config, head='ocata_expand01')
+
+    def _pre_upgrade_ocata_expand01(self, engine):
+        images = db_utils.get_table(engine, 'images')
+        image_members = db_utils.get_table(engine, 'image_members')
+        now = datetime.datetime.now()
+
+        # inserting a public image record
+        public_temp = dict(deleted=False,
+                           created_at=now,
+                           status='active',
+                           is_public=True,
+                           min_disk=0,
+                           min_ram=0,
+                           id='public_id')
+        images.insert().values(public_temp).execute()
+
+        # inserting a non-public image record for 'shared' visibility test
+        shared_temp = dict(deleted=False,
+                           created_at=now,
+                           status='active',
+                           is_public=False,
+                           min_disk=0,
+                           min_ram=0,
+                           id='shared_id')
+        images.insert().values(shared_temp).execute()
+
+        # inserting a non-public image records for 'private' visibility test
+        private_temp = dict(deleted=False,
+                            created_at=now,
+                            status='active',
+                            is_public=False,
+                            min_disk=0,
+                            min_ram=0,
+                            id='private_id_1')
+        images.insert().values(private_temp).execute()
+
+        private_temp = dict(deleted=False,
+                            created_at=now,
+                            status='active',
+                            is_public=False,
+                            min_disk=0,
+                            min_ram=0,
+                            id='private_id_2')
+        images.insert().values(private_temp).execute()
+
+        # adding an active as well as a deleted image member for checking
+        # 'shared' visibility
+        temp = dict(deleted=False,
+                    created_at=now,
+                    image_id='shared_id',
+                    member='fake_member_452',
+                    can_share=True,
+                    id=45)
+        image_members.insert().values(temp).execute()
+
+        temp = dict(deleted=True,
+                    created_at=now,
+                    image_id='shared_id',
+                    member='fake_member_453',
+                    can_share=True,
+                    id=453)
+        image_members.insert().values(temp).execute()
+
+        # adding an image member, but marking it deleted,
+        # for testing 'private' visibility
+        temp = dict(deleted=True,
+                    created_at=now,
+                    image_id='private_id_2',
+                    member='fake_member_451',
+                    can_share=True,
+                    id=451)
+        image_members.insert().values(temp).execute()
+
+        # adding an active image member for the 'public' image,
+        # to test it remains public regardless.
+        temp = dict(deleted=False,
+                    created_at=now,
+                    image_id='public_id',
+                    member='fake_member_450',
+                    can_share=True,
+                    id=450)
+        image_members.insert().values(temp).execute()
+
+    def _check_ocata_expand01(self, engine, data):
+        images = db_utils.get_table(engine, 'images')
+
+        # check that visibility is null for existing images
+        rows = (images.select()
+                .order_by(images.c.id)
+                .execute()
+                .fetchall())
+        self.assertEqual(4, len(rows))
+        for row in rows:
+            self.assertIsNone(row['visibility'])
+
+        # run data migrations
+        data_migrations.migrate(engine)
+
+        # check that visibility is set appropriately for all images
+        rows = (images.select()
+                .order_by(images.c.id)
+                .execute()
+                .fetchall())
+        self.assertEqual(4, len(rows))
+        # private_id_1 has private visibility
+        self.assertEqual('private_id_1', rows[0]['id'])
+        self.assertEqual('private', rows[0]['visibility'])
+        # private_id_2 has private visibility
+        self.assertEqual('private_id_2', rows[1]['id'])
+        self.assertEqual('private', rows[1]['visibility'])
+        # public_id has public visibility
+        self.assertEqual('public_id', rows[2]['id'])
+        self.assertEqual('public', rows[2]['visibility'])
+        # shared_id has shared visibility
+        self.assertEqual('shared_id', rows[3]['id'])
+        self.assertEqual('shared', rows[3]['visibility'])
+
+
+class TestOcataMigrate01MySQL(TestOcataMigrate01Mixin,
+                              test_base.MySQLOpportunisticTestCase):
+    pass

glance/tests/functional/db/test_migrations.py

@@ -23,7 +23,7 @@ from oslo_db.sqlalchemy import test_base
 from oslo_db.sqlalchemy import test_migrations
 import sqlalchemy.types as types
 
-from glance.db import migration as dm
+from glance.db import migration as db_migration
 from glance.db.sqlalchemy import alembic_migrations
 from glance.db.sqlalchemy.alembic_migrations import versions
 from glance.db.sqlalchemy import models
@@ -34,10 +34,11 @@ import glance.tests.utils as test_utils
 
 class AlembicMigrationsMixin(object):
 
-    def _get_revisions(self, config):
+    def _get_revisions(self, config, head=None):
+        head = head or db_migration.LATEST_REVISION
         scripts_dir = alembic_script.ScriptDirectory.from_config(config)
         revisions = list(scripts_dir.walk_revisions(base='base',
-                                                    head=dm.LATEST_REVISION))
+                                                    head=head))
         revisions = list(reversed(revisions))
         revisions = [rev.revision for rev in revisions]
         return revisions