Refactor tests to use Alembic to run migrations

* Functional tests now use alembic instead of sqlalchmey-migrate
  to build and destroy test database.
* All tests now use a file-based sqlite db as opposed to an in-memory
  database.

Partially-Implements: blueprint alembic-migrations
Change-Id: I77921366a05ba6f9841143af89c1f4059d8454c6
Depends-On: Ie8594ff339a13bf190aefa308f54e97ee20ecfa2

glance/db/migration.py

@@ -26,8 +26,6 @@ from oslo_config import cfg
 from oslo_db import options as db_options
 from stevedore import driver
 
-from glance.db.sqlalchemy import api as db_api
-
 
 _IMPL = None
 _LOCK = threading.Lock()
@@ -53,14 +51,3 @@ MIGRATE_REPO_PATH = os.path.join(
     'sqlalchemy',
     'migrate_repo',
 )
-
-
-def db_sync(version=None, init_version=0, engine=None):
-    """Migrate the database to `version` or the most recent version."""
-
-    if engine is None:
-        engine = db_api.get_engine()
-    return get_backend().db_sync(engine=engine,
-                                 abs_path=MIGRATE_REPO_PATH,
-                                 version=version,
-                                 init_version=init_version)

glance/db/sqlalchemy/alembic_migrations/__init__.py

@@ -20,19 +20,20 @@ from alembic import command as alembic_command
 from alembic import config as alembic_config
 from alembic import migration as alembic_migration
 from oslo_db import exception as db_exception
-from oslo_db.sqlalchemy import migration
+from oslo_db.sqlalchemy import migration as sqla_migration
 
 from glance.db import migration as db_migration
 from glance.db.sqlalchemy import api as db_api
 from glance.i18n import _
 
 
-def get_alembic_config():
+def get_alembic_config(engine=None):
     """Return a valid alembic config object"""
     ini_path = os.path.join(os.path.dirname(__file__), 'alembic.ini')
     config = alembic_config.Config(os.path.abspath(ini_path))
-    dbconn = str(db_api.get_engine().url)
+    if engine is None:
-    config.set_main_option('sqlalchemy.url', dbconn)
+        engine = db_api.get_engine()
+    config.set_main_option('sqlalchemy.url', str(engine.url))
     return config
 
 
@@ -47,9 +48,9 @@ def get_current_alembic_heads():
 
 def get_current_legacy_head():
     try:
-        legacy_head = migration.db_version(db_api.get_engine(),
+        legacy_head = sqla_migration.db_version(db_api.get_engine(),
-                                           db_migration.MIGRATE_REPO_PATH,
+                                                db_migration.MIGRATE_REPO_PATH,
-                                           db_migration.INIT_VERSION)
+                                                db_migration.INIT_VERSION)
     except db_exception.DbMigrationError:
         legacy_head = None
     return legacy_head

glance/tests/functional/db/migrations/test_mitaka01.py

@@ -0,0 +1,48 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_db.sqlalchemy import test_base
+import sqlalchemy
+
+from glance.tests.functional.db import test_migrations
+
+
+def get_indexes(table, engine):
+    inspector = sqlalchemy.inspect(engine)
+    return [idx['name'] for idx in inspector.get_indexes(table)]
+
+
+class TestMitaka01Mixin(test_migrations.AlembicMigrationsMixin):
+
+    def _pre_upgrade_mitaka01(self, engine):
+        indexes = get_indexes('images', engine)
+        self.assertNotIn('created_at_image_idx', indexes)
+        self.assertNotIn('updated_at_image_idx', indexes)
+
+    def _check_mitaka01(self, engine, data):
+        indexes = get_indexes('images', engine)
+        self.assertIn('created_at_image_idx', indexes)
+        self.assertIn('updated_at_image_idx', indexes)
+
+
+class TestMitaka01MySQL(TestMitaka01Mixin,
+                        test_base.MySQLOpportunisticTestCase):
+    pass
+
+
+class TestMitaka01PostgresSQL(TestMitaka01Mixin,
+                              test_base.PostgreSQLOpportunisticTestCase):
+    pass
+
+
+class TestMitaka01Sqlite(TestMitaka01Mixin, test_base.DbTestCase):
+    pass

glance/tests/functional/db/migrations/test_mitaka02.py

@@ -0,0 +1,65 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import datetime
+
+from oslo_db.sqlalchemy import test_base
+from oslo_db.sqlalchemy import utils as db_utils
+
+from glance.tests.functional.db import test_migrations
+
+
+class TestMitaka02Mixin(test_migrations.AlembicMigrationsMixin):
+
+    def _pre_upgrade_mitaka02(self, engine):
+        metadef_resource_types = db_utils.get_table(engine,
+                                                    'metadef_resource_types')
+        now = datetime.datetime.now()
+        db_rec1 = dict(id='9580',
+                       name='OS::Nova::Instance',
+                       protected=False,
+                       created_at=now,
+                       updated_at=now,)
+        db_rec2 = dict(id='9581',
+                       name='OS::Nova::Blah',
+                       protected=False,
+                       created_at=now,
+                       updated_at=now,)
+        db_values = (db_rec1, db_rec2)
+        metadef_resource_types.insert().values(db_values).execute()
+
+    def _check_mitaka02(self, engine, data):
+        metadef_resource_types = db_utils.get_table(engine,
+                                                    'metadef_resource_types')
+        result = (metadef_resource_types.select()
+                  .where(metadef_resource_types.c.name == 'OS::Nova::Instance')
+                  .execute().fetchall())
+        self.assertEqual(0, len(result))
+
+        result = (metadef_resource_types.select()
+                  .where(metadef_resource_types.c.name == 'OS::Nova::Server')
+                  .execute().fetchall())
+        self.assertEqual(1, len(result))
+
+
+class TestMitaka02MySQL(TestMitaka02Mixin,
+                        test_base.MySQLOpportunisticTestCase):
+    pass
+
+
+class TestMitaka02PostgresSQL(TestMitaka02Mixin,
+                              test_base.PostgreSQLOpportunisticTestCase):
+    pass
+
+
+class TestMitaka02Sqlite(TestMitaka02Mixin, test_base.DbTestCase):
+    pass

glance/tests/functional/db/migrations/test_ocata01.py

@@ -0,0 +1,142 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import datetime
+
+from oslo_db.sqlalchemy import test_base
+from oslo_db.sqlalchemy import utils as db_utils
+
+from glance.tests.functional.db import test_migrations
+
+
+class TestOcata01Mixin(test_migrations.AlembicMigrationsMixin):
+
+    def _pre_upgrade_ocata01(self, engine):
+        images = db_utils.get_table(engine, 'images')
+        now = datetime.datetime.now()
+        image_members = db_utils.get_table(engine, 'image_members')
+
+        # inserting a public image record
+        public_temp = dict(deleted=False,
+                           created_at=now,
+                           status='active',
+                           is_public=True,
+                           min_disk=0,
+                           min_ram=0,
+                           id='public_id')
+        images.insert().values(public_temp).execute()
+
+        # inserting a non-public image record for 'shared' visibility test
+        shared_temp = dict(deleted=False,
+                           created_at=now,
+                           status='active',
+                           is_public=False,
+                           min_disk=0,
+                           min_ram=0,
+                           id='shared_id')
+        images.insert().values(shared_temp).execute()
+
+        # inserting a non-public image records for 'private' visibility test
+        private_temp = dict(deleted=False,
+                            created_at=now,
+                            status='active',
+                            is_public=False,
+                            min_disk=0,
+                            min_ram=0,
+                            id='private_id_1')
+        images.insert().values(private_temp).execute()
+        private_temp = dict(deleted=False,
+                            created_at=now,
+                            status='active',
+                            is_public=False,
+                            min_disk=0,
+                            min_ram=0,
+                            id='private_id_2')
+        images.insert().values(private_temp).execute()
+
+        # adding an active as well as a deleted image member for checking
+        # 'shared' visibility
+        temp = dict(deleted=False,
+                    created_at=now,
+                    image_id='shared_id',
+                    member='fake_member_452',
+                    can_share=True,
+                    id=45)
+        image_members.insert().values(temp).execute()
+
+        temp = dict(deleted=True,
+                    created_at=now,
+                    image_id='shared_id',
+                    member='fake_member_453',
+                    can_share=True,
+                    id=453)
+        image_members.insert().values(temp).execute()
+
+        # adding an image member, but marking it deleted,
+        # for testing 'private' visibility
+        temp = dict(deleted=True,
+                    created_at=now,
+                    image_id='private_id_2',
+                    member='fake_member_451',
+                    can_share=True,
+                    id=451)
+        image_members.insert().values(temp).execute()
+
+        # adding an active image member for the 'public' image,
+        # to test it remains public regardless.
+        temp = dict(deleted=False,
+                    created_at=now,
+                    image_id='public_id',
+                    member='fake_member_450',
+                    can_share=True,
+                    id=450)
+        image_members.insert().values(temp).execute()
+
+    def _check_ocata01(self, engine, data):
+        # check that after migration, 'visibility' column is introduced
+        images = db_utils.get_table(engine, 'images')
+        self.assertIn('visibility', images.c)
+        self.assertNotIn('is_public', images.c)
+
+        # tests to identify the visibilities of images created above
+        rows = images.select().where(
+            images.c.id == 'public_id').execute().fetchall()
+        self.assertEqual(1, len(rows))
+        self.assertEqual('public', rows[0][16])
+
+        rows = images.select().where(
+            images.c.id == 'shared_id').execute().fetchall()
+        self.assertEqual(1, len(rows))
+        self.assertEqual('shared', rows[0][16])
+
+        rows = images.select().where(
+            images.c.id == 'private_id_1').execute().fetchall()
+        self.assertEqual(1, len(rows))
+        self.assertEqual('private', rows[0][16])
+
+        rows = images.select().where(
+            images.c.id == 'private_id_2').execute().fetchall()
+        self.assertEqual(1, len(rows))
+        self.assertEqual('private', rows[0][16])
+
+
+class TestOcata01MySQL(TestOcata01Mixin, test_base.MySQLOpportunisticTestCase):
+    pass
+
+
+class TestOcata01PostgresSQL(TestOcata01Mixin,
+                             test_base.PostgreSQLOpportunisticTestCase):
+    pass
+
+
+class TestOcata01Sqlite(TestOcata01Mixin, test_base.DbTestCase):
+    pass

glance/tests/functional/db/test_migrations.py

@@ -0,0 +1,173 @@
+# Copyright 2016 Rackspace
+# Copyright 2016 Intel Corporation
+#
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+from alembic import command as alembic_command
+from alembic import script as alembic_script
+from oslo_db.sqlalchemy import test_base
+from oslo_db.sqlalchemy import test_migrations
+import sqlalchemy.types as types
+
+from glance.db.sqlalchemy import alembic_migrations
+from glance.db.sqlalchemy.alembic_migrations import versions
+from glance.db.sqlalchemy import models
+from glance.db.sqlalchemy import models_glare
+from glance.db.sqlalchemy import models_metadef
+import glance.tests.utils as test_utils
+
+
+class AlembicMigrationsMixin(object):
+
+    def _get_revisions(self, config):
+        scripts_dir = alembic_script.ScriptDirectory.from_config(config)
+        revisions = list(scripts_dir.walk_revisions(base='base', head='heads'))
+        revisions = list(reversed(revisions))
+        revisions = [rev.revision for rev in revisions]
+        return revisions
+
+    def _migrate_up(self, config, engine, revision, with_data=False):
+        if with_data:
+            data = None
+            pre_upgrade = getattr(self, '_pre_upgrade_%s' % revision, None)
+            if pre_upgrade:
+                data = pre_upgrade(engine)
+
+        alembic_command.upgrade(config, revision)
+
+        if with_data:
+            check = getattr(self, '_check_%s' % revision, None)
+            if check:
+                check(engine, data)
+
+    def test_walk_versions(self):
+        alembic_config = alembic_migrations.get_alembic_config(self.engine)
+        for revision in self._get_revisions(alembic_config):
+            self._migrate_up(alembic_config, self.engine, revision,
+                             with_data=True)
+
+
+class TestMysqlMigrations(test_base.MySQLOpportunisticTestCase,
+                          AlembicMigrationsMixin):
+
+    def test_mysql_innodb_tables(self):
+        test_utils.db_sync(engine=self.engine)
+
+        total = self.engine.execute(
+            "SELECT COUNT(*) "
+            "FROM information_schema.TABLES "
+            "WHERE TABLE_SCHEMA='%s'"
+            % self.engine.url.database)
+        self.assertGreater(total.scalar(), 0, "No tables found. Wrong schema?")
+
+        noninnodb = self.engine.execute(
+            "SELECT count(*) "
+            "FROM information_schema.TABLES "
+            "WHERE TABLE_SCHEMA='%s' "
+            "AND ENGINE!='InnoDB' "
+            "AND TABLE_NAME!='migrate_version'"
+            % self.engine.url.database)
+        count = noninnodb.scalar()
+        self.assertEqual(0, count, "%d non InnoDB tables created" % count)
+
+
+class TestPostgresqlMigrations(test_base.PostgreSQLOpportunisticTestCase,
+                               AlembicMigrationsMixin):
+    pass
+
+
+class TestSqliteMigrations(test_base.DbTestCase, AlembicMigrationsMixin):
+    pass
+
+
+class TestMigrations(test_base.DbTestCase, test_utils.BaseTestCase):
+
+    def test_no_downgrade(self):
+        migrate_file = versions.__path__[0]
+        for parent, dirnames, filenames in os.walk(migrate_file):
+            for filename in filenames:
+                if filename.split('.')[1] == 'py':
+                    model_name = filename.split('.')[0]
+                    model = __import__(
+                        'glance.db.sqlalchemy.alembic_migrations.versions.' +
+                        model_name)
+                    obj = getattr(getattr(getattr(getattr(getattr(
+                        model, 'db'), 'sqlalchemy'), 'alembic_migrations'),
+                        'versions'), model_name)
+                    func = getattr(obj, 'downgrade', None)
+                    self.assertIsNone(func)
+
+
+class ModelsMigrationSyncMixin(object):
+
+    def get_metadata(self):
+        for table in models_metadef.BASE_DICT.metadata.sorted_tables:
+            models.BASE.metadata._add_table(table.name, table.schema, table)
+        for table in models_glare.BASE.metadata.sorted_tables:
+            models.BASE.metadata._add_table(table.name, table.schema, table)
+        return models.BASE.metadata
+
+    def get_engine(self):
+        return self.engine
+
+    def db_sync(self, engine):
+        test_utils.db_sync(engine=engine)
+
+    # TODO(akamyshikova): remove this method as soon as comparison with Variant
+    # will be implemented in oslo.db or alembic
+    def compare_type(self, ctxt, insp_col, meta_col, insp_type, meta_type):
+        if isinstance(meta_type, types.Variant):
+            meta_orig_type = meta_col.type
+            insp_orig_type = insp_col.type
+            meta_col.type = meta_type.impl
+            insp_col.type = meta_type.impl
+
+            try:
+                return self.compare_type(ctxt, insp_col, meta_col, insp_type,
+                                         meta_type.impl)
+            finally:
+                meta_col.type = meta_orig_type
+                insp_col.type = insp_orig_type
+        else:
+            ret = super(ModelsMigrationSyncMixin, self).compare_type(
+                ctxt, insp_col, meta_col, insp_type, meta_type)
+            if ret is not None:
+                return ret
+            return ctxt.impl.compare_type(insp_col, meta_col)
+
+    def include_object(self, object_, name, type_, reflected, compare_to):
+        if name in ['migrate_version'] and type_ == 'table':
+            return False
+        return True
+
+
+class ModelsMigrationsSyncMysql(ModelsMigrationSyncMixin,
+                                test_migrations.ModelsMigrationsSync,
+                                test_base.MySQLOpportunisticTestCase):
+    pass
+
+
+class ModelsMigrationsSyncPostgres(ModelsMigrationSyncMixin,
+                                   test_migrations.ModelsMigrationsSync,
+                                   test_base.PostgreSQLOpportunisticTestCase):
+    pass
+
+
+class ModelsMigrationsSyncSqlite(ModelsMigrationSyncMixin,
+                                 test_migrations.ModelsMigrationsSync,
+                                 test_base.DbTestCase):
+    pass

glance/tests/integration/legacy_functional/base.py

@@ -21,7 +21,6 @@ from oslo_db import options
 
 import glance.common.client
 from glance.common import config
-from glance.db import migration
 import glance.db.sqlalchemy.api
 import glance.registry.client.v1.client
 from glance import tests as glance_tests
@@ -171,7 +170,7 @@ class ApiTest(test_utils.BaseTestCase):
             test_utils.execute('cp %s %s/tests.sqlite'
                                % (db_location, self.test_dir))
         else:
-            migration.db_sync()
+            test_utils.db_sync()
 
             # copy the clean db to a temp location so that it
             # can be reused for future tests

glance/tests/integration/v2/base.py

@@ -24,7 +24,6 @@ from oslo_db import options
 
 import glance.common.client
 from glance.common import config
-from glance.db import migration
 import glance.db.sqlalchemy.api
 import glance.registry.client.v1.client
 from glance import tests as glance_tests
@@ -166,7 +165,7 @@ class ApiTest(test_utils.BaseTestCase):
             test_utils.execute('cp %s %s/tests.sqlite'
                                % (db_location, self.test_dir))
         else:
-            migration.db_sync()
+            test_utils.db_sync()
 
             # copy the clean db to a temp location so that it
             # can be reused for future tests

glance/tests/utils.py

@@ -23,6 +23,7 @@ import shutil
 import socket
 import subprocess
 
+from alembic import command as alembic_command
 import fixtures
 from oslo_config import cfg
 from oslo_config import fixture as cfg_fixture
@@ -42,6 +43,7 @@ from glance.common import timeutils
 from glance.common import utils
 from glance.common import wsgi
 from glance import context
+from glance.db.sqlalchemy import alembic_migrations
 from glance.db.sqlalchemy import api as db_api
 from glance.db.sqlalchemy import models as db_models
 
@@ -670,3 +672,14 @@ class HttplibWsgiAdapter(object):
         response = self.req.get_response(self.app)
         return FakeHTTPResponse(response.status_code, response.headers,
                                 response.body)
+
+
+def db_sync(version=None, engine=None):
+    """Migrate the database to `version` or the most recent version."""
+    if version is None:
+        version = 'heads'
+    if engine is None:
+        engine = db_api.get_engine()
+
+    alembic_config = alembic_migrations.get_alembic_config(engine=engine)
+    alembic_command.upgrade(alembic_config, version)

tox.ini

@@ -10,6 +10,15 @@ basepython =
 setenv =
   VIRTUAL_ENV={envdir}
   PYTHONWARNINGS=default::DeprecationWarning
+# NOTE(hemanthm): The environment variable "OS_TEST_DBAPI_ADMIN_CONNECTION"
+# must be set to force oslo.db tests to use a file-based sqlite database
+# instead of the default in-memory database, which doesn't work well with
+# alembic migrations. The file-based database pointed by the environment
+# variable itself is not used for testing. Neither is it ever created. Oslo.db
+# creates another file-based database for testing purposes and deletes it as a
+# part of its test clean-up. Think of setting this environment variable as a
+# clue for oslo.db to use file-based database.
+  OS_TEST_DBAPI_ADMIN_CONNECTION=sqlite:////tmp/placeholder-never-created-nor-used.db
 usedevelop = True
 install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
 deps = -r{toxinidir}/test-requirements.txt