diff --git a/glance/db/migration.py b/glance/db/migration.py index 5ac5efbd8e..86adf3588f 100644 --- a/glance/db/migration.py +++ b/glance/db/migration.py @@ -21,9 +21,16 @@ import os +from oslo.config import cfg +from oslo import db + from glance.common import utils from glance.db.sqlalchemy import api as db_api + +db.options.set_defaults(cfg.CONF) + + IMPL = utils.LazyPluggable( 'backend', config_group='database', @@ -38,9 +45,11 @@ MIGRATE_REPO_PATH = os.path.join( ) -def db_sync(version=None, init_version=0): +def db_sync(version=None, init_version=0, engine=None): """Migrate the database to `version` or the most recent version.""" - return IMPL.db_sync(engine=db_api.get_engine(), + if engine is None: + engine = db_api.get_engine() + return IMPL.db_sync(engine=engine, abs_path=MIGRATE_REPO_PATH, version=version, init_version=init_version) diff --git a/glance/db/sqlalchemy/migrate_repo/versions/012_id_to_uuid.py b/glance/db/sqlalchemy/migrate_repo/versions/012_id_to_uuid.py index b506b0ad9a..5d0d82807e 100644 --- a/glance/db/sqlalchemy/migrate_repo/versions/012_id_to_uuid.py +++ b/glance/db/sqlalchemy/migrate_repo/versions/012_id_to_uuid.py @@ -456,9 +456,23 @@ def _downgrade_other(t_images, t_image_members, t_image_properties, dialect): _update_all_uuids_to_ids(t_images, t_image_members, t_image_properties) - t_images.c.id.alter(sqlalchemy.Integer(), primary_key=True) - t_image_members.c.image_id.alter(sqlalchemy.Integer()) - t_image_properties.c.image_id.alter(sqlalchemy.Integer()) + t_images.c.id.alter(primary_key=True) + #we have to use raw sql for postgresql as we have errors + #if we use alter type on sqlalchemy + if dialect == 'postgresql': + t_images.bind.execute('''ALTER TABLE images + ALTER COLUMN id TYPE INTEGER + USING (id::INTEGER)''') + t_images.bind.execute('''ALTER TABLE image_members + ALTER COLUMN image_id TYPE INTEGER + USING (image_id::INTEGER)''') + t_images.bind.execute('''ALTER TABLE image_properties + ALTER COLUMN image_id TYPE INTEGER + USING (image_id::INTEGER)''') + else: + t_images.c.id.alter(sqlalchemy.Integer()) + t_image_members.c.image_id.alter(sqlalchemy.Integer()) + t_image_properties.c.image_id.alter(sqlalchemy.Integer()) for fk in foreign_keys: fk.create() diff --git a/glance/tests/unit/test_migrations.conf b/glance/tests/unit/test_migrations.conf deleted file mode 100644 index d7e14f0947..0000000000 --- a/glance/tests/unit/test_migrations.conf +++ /dev/null @@ -1,8 +0,0 @@ -[DEFAULT] -# Set up any number of migration data stores you want, one -# The "name" used in the test is the config variable key. -sqlite=sqlite:///test_migrations.db -#mysql=mysql://root:@localhost/test_migrations -#postgresql=postgresql://root:@localhost/test_migrations -[walk_style] -snake_walk=yes diff --git a/glance/tests/unit/test_migrations.py b/glance/tests/unit/test_migrations.py index acac4bafbc..ee6e08bbe6 100644 --- a/glance/tests/unit/test_migrations.py +++ b/glance/tests/unit/test_migrations.py @@ -25,381 +25,54 @@ if possible. from __future__ import print_function -import ConfigParser import datetime import exceptions import os import pickle -import subprocess import uuid from migrate.versioning import api as migration_api from migrate.versioning.repository import Repository from oslo.config import cfg -import six.moves.urllib.parse as urlparse -from six.moves import xrange -from six import text_type +from oslo.db.sqlalchemy import test_base +from oslo.db.sqlalchemy import test_migrations +from oslo.db.sqlalchemy import utils as db_utils import sqlalchemy from glance.common import crypt from glance.common import exception from glance.common import utils -import glance.db.migration as migration -import glance.db.sqlalchemy.migrate_repo +from glance.db import migration +from glance.db.sqlalchemy import migrate_repo from glance.db.sqlalchemy.migrate_repo.schema import from_migration_import from glance.db.sqlalchemy import models from glance.openstack.common import jsonutils -from glance.openstack.common import log as logging from glance.openstack.common import timeutils -from glance.tests import utils as test_utils - - CONF = cfg.CONF CONF.import_opt('metadata_encryption_key', 'glance.common.config') -LOG = logging.getLogger(__name__) +class MigrationsMixin(test_migrations.WalkVersionsMixin): + @property + def INIT_VERSION(self): + return migration.INIT_VERSION -def _get_connect_string(backend, - user="openstack_citest", - passwd="openstack_citest", - database="openstack_citest"): - """ - Try to get a connection with a very specific set of values, if we get - these then we'll run the tests, otherwise they are skipped - """ - if backend == "mysql": - backend = "mysql+mysqldb" - elif backend == "postgres": - backend = "postgresql+psycopg2" + @property + def REPOSITORY(self): + migrate_file = migrate_repo.__file__ + return Repository(os.path.abspath(os.path.dirname(migrate_file))) - return ("%(backend)s://%(user)s:%(passwd)s@localhost/%(database)s" - % {'backend': backend, 'user': user, 'passwd': passwd, - 'database': database}) + @property + def migration_api(self): + return migration_api - -def _is_backend_avail(backend, - user="openstack_citest", - passwd="openstack_citest", - database="openstack_citest"): - try: - if backend == "mysql": - connect_uri = _get_connect_string("mysql", user=user, - passwd=passwd, database=database) - elif backend == "postgres": - connect_uri = _get_connect_string("postgres", user=user, - passwd=passwd, database=database) - engine = sqlalchemy.create_engine(connect_uri) - connection = engine.connect() - except Exception: - # intentionally catch all to handle exceptions even if we don't - # have any backend code loaded. - return False - else: - connection.close() - engine.dispose() - return True - - -def _have_mysql(): - present = os.environ.get('GLANCE_TEST_MYSQL_PRESENT') - if present is None: - return _is_backend_avail('mysql') - return present.lower() in ('', 'true') - - -def get_table(engine, name): - """Returns an sqlalchemy table dynamically from db. - - Needed because the models don't work for us in migrations - as models will be far out of sync with the current data. - """ - metadata = sqlalchemy.schema.MetaData() - metadata.bind = engine - return sqlalchemy.Table(name, metadata, autoload=True) - - -class TestMigrations(test_utils.BaseTestCase): - """Test sqlalchemy-migrate migrations.""" - - DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), - 'test_migrations.conf') - # Test machines can set the GLANCE_TEST_MIGRATIONS_CONF variable - # to override the location of the config file for migration testing - CONFIG_FILE_PATH = os.environ.get('GLANCE_TEST_MIGRATIONS_CONF', - DEFAULT_CONFIG_FILE) - MIGRATE_FILE = glance.db.sqlalchemy.migrate_repo.__file__ - REPOSITORY = Repository(os.path.abspath(os.path.dirname(MIGRATE_FILE))) - - def setUp(self): - super(TestMigrations, self).setUp() - - self.snake_walk = False - self.test_databases = {} - - # Load test databases from the config file. Only do this - # once. No need to re-run this on each test... - LOG.debug('config_path is %s', - text_type(TestMigrations.CONFIG_FILE_PATH)) - if os.path.exists(TestMigrations.CONFIG_FILE_PATH): - cp = ConfigParser.RawConfigParser() - try: - cp.read(TestMigrations.CONFIG_FILE_PATH) - defaults = cp.defaults() - for key, value in defaults.items(): - self.test_databases[key] = value - self.snake_walk = cp.getboolean('walk_style', 'snake_walk') - except ConfigParser.ParsingError as e: - self.fail("Failed to read test_migrations.conf config " - "file. Got error: %s" % e) - else: - self.fail("Failed to find test_migrations.conf config " - "file.") - - self.engines = {} - for key, value in self.test_databases.items(): - self.engines[key] = sqlalchemy.create_engine(value) - - # We start each test case with a completely blank slate. - self._reset_databases() - - def tearDown(self): - # We destroy the test data store between each test case, - # and recreate it, which ensures that we have no side-effects - # from the tests - self._reset_databases() - super(TestMigrations, self).tearDown() - - def _reset_databases(self): - def execute_cmd(cmd=None): - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, shell=True) - output = proc.communicate()[0] - LOG.debug(output) - self.assertEqual(0, proc.returncode) - - for key, engine in self.engines.items(): - conn_string = self.test_databases[key] - conn_pieces = urlparse.urlparse(conn_string) - engine.dispose() - if conn_string.startswith('sqlite'): - # We can just delete the SQLite database, which is - # the easiest and cleanest solution - db_path = conn_pieces.path[1:] - if os.path.exists(db_path): - os.unlink(db_path) - # No need to recreate the SQLite DB. SQLite will - # create it for us if it's not there... - elif conn_string.startswith('mysql'): - # We can execute the MySQL client to destroy and re-create - # the MYSQL database, which is easier and less error-prone - # than using SQLAlchemy to do this via MetaData...trust me. - database = conn_pieces.path.strip('/') - loc_pieces = conn_pieces.netloc.split('@') - host = loc_pieces[1] - auth_pieces = loc_pieces[0].split(':') - user = auth_pieces[0] - password = "" - if len(auth_pieces) > 1: - if auth_pieces[1].strip(): - password = "-p\"%s\"" % auth_pieces[1] - sql = ("drop database if exists %(database)s; create " - "database %(database)s;") % {'database': database} - cmd = ("mysql -u \"%(user)s\" %(password)s -h %(host)s " - "-e \"%(sql)s\"") % {'user': user, 'password': password, - 'host': host, 'sql': sql} - execute_cmd(cmd) - elif conn_string.startswith('postgresql'): - database = conn_pieces.path.strip('/') - loc_pieces = conn_pieces.netloc.split('@') - host = loc_pieces[1] - - auth_pieces = loc_pieces[0].split(':') - user = auth_pieces[0] - password = "" - if len(auth_pieces) > 1: - password = auth_pieces[1].strip() - # note(boris-42): This file is used for authentication - # without password prompt. - createpgpass = ("echo '*:*:*:%(user)s:%(password)s' > " - "~/.pgpass && chmod 0600 ~/.pgpass" % - {'user': user, 'password': password}) - execute_cmd(createpgpass) - # note(boris-42): We must create and drop database, we can't - # drop database which we have connected to, so for such - # operations there is a special database template1. - sqlcmd = ("psql -w -U %(user)s -h %(host)s -c" - " '%(sql)s' -d template1") - sql = ("drop database if exists %(database)s;") - sql = sql % {'database': database} - droptable = sqlcmd % {'user': user, 'host': host, - 'sql': sql} - execute_cmd(droptable) - sql = ("create database %(database)s;") - sql = sql % {'database': database} - createtable = sqlcmd % {'user': user, 'host': host, - 'sql': sql} - execute_cmd(createtable) + @property + def migrate_engine(self): + return self.engine def test_walk_versions(self): - """ - Walks all version scripts for each tested database, ensuring - that there are no errors in the version scripts for each engine - """ - for key, engine in self.engines.items(): - self._walk_versions(engine, self.snake_walk) - - def test_mysql_connect_fail(self): - """ - Test that we can trigger a mysql connection failure and we fail - gracefully to ensure we don't break people without mysql - """ - if _is_backend_avail('mysql', user="openstack_cifail"): - self.fail("Shouldn't have connected") - - def test_mysql_opportunistically(self): - # Test that table creation on mysql only builds InnoDB tables - if not _is_backend_avail('mysql'): - self.skipTest("mysql not available") - # add this to the global lists to make reset work with it, it's removed - # automatically in tearDown so no need to clean it up here. - connect_string = _get_connect_string("mysql") - engine = sqlalchemy.create_engine(connect_string) - self.engines["mysqlcitest"] = engine - self.test_databases["mysqlcitest"] = connect_string - - # build a fully populated mysql database with all the tables - self._reset_databases() - self._walk_versions(engine, False, False) - - connection = engine.connect() - # sanity check - total = connection.execute("SELECT count(*) " - "from information_schema.TABLES " - "where TABLE_SCHEMA='openstack_citest'") - self.assertTrue(total.scalar() > 0, "No tables found. Wrong schema?") - - noninnodb = connection.execute("SELECT count(*) " - "from information_schema.TABLES " - "where TABLE_SCHEMA='openstack_citest' " - "and ENGINE!='InnoDB' " - "and TABLE_NAME!='migrate_version'") - count = noninnodb.scalar() - self.assertEqual(count, 0, "%d non InnoDB tables created" % count) - connection.close() - - def test_postgresql_connect_fail(self): - """ - Test that we can trigger a postgres connection failure and we fail - gracefully to ensure we don't break people without postgres - """ - if _is_backend_avail('postgresql', user="openstack_cifail"): - self.fail("Shouldn't have connected") - - def test_postgresql_opportunistically(self): - # Test postgresql database migration walk - if not _is_backend_avail('postgres'): - self.skipTest("postgresql not available") - # add this to the global lists to make reset work with it, it's removed - # automatically in tearDown so no need to clean it up here. - connect_string = _get_connect_string("postgres") - engine = sqlalchemy.create_engine(connect_string) - self.engines["postgresqlcitest"] = engine - self.test_databases["postgresqlcitest"] = connect_string - - # build a fully populated postgresql database with all the tables - self._reset_databases() - self._walk_versions(engine, False, False) - - def _walk_versions(self, engine=None, snake_walk=False, downgrade=True, - initial_version=None): - # Determine latest version script from the repo, then - # upgrade from 1 through to the latest, with no data - # in the databases. This just checks that the schema itself - # upgrades successfully. - - def db_version(): - return migration_api.db_version(engine, TestMigrations.REPOSITORY) - - # Place the database under version control - init_version = migration.INIT_VERSION - if initial_version is not None: - init_version = initial_version - migration_api.version_control(engine, TestMigrations.REPOSITORY, - init_version) - self.assertEqual(init_version, db_version()) - - migration_api.upgrade(engine, TestMigrations.REPOSITORY, - init_version + 1) - self.assertEqual(init_version + 1, db_version()) - - LOG.debug('latest version is %s', TestMigrations.REPOSITORY.latest) - - for version in xrange(init_version + 2, - TestMigrations.REPOSITORY.latest + 1): - # upgrade -> downgrade -> upgrade - self._migrate_up(engine, version, with_data=True) - if snake_walk: - self._migrate_down(engine, version - 1, with_data=True) - self._migrate_up(engine, version) - - if downgrade: - # Now walk it back down to 0 from the latest, testing - # the downgrade paths. - for version in reversed( - xrange(init_version + 2, - TestMigrations.REPOSITORY.latest + 1)): - # downgrade -> upgrade -> downgrade - self._migrate_down(engine, version - 1) - if snake_walk: - self._migrate_up(engine, version) - self._migrate_down(engine, version - 1) - - # Ensure we made it all the way back to the first migration - self.assertEqual(init_version + 1, db_version()) - - def _migrate_down(self, engine, version, with_data=False): - migration_api.downgrade(engine, - TestMigrations.REPOSITORY, - version) - self.assertEqual(version, - migration_api.db_version(engine, - TestMigrations.REPOSITORY)) - - # NOTE(sirp): `version` is what we're downgrading to (i.e. the 'target' - # version). So if we have any downgrade checks, they need to be run for - # the previous (higher numbered) migration. - if with_data: - post_downgrade = getattr(self, "_post_downgrade_%03d" % - (version + 1), None) - if post_downgrade: - post_downgrade(engine) - - def _migrate_up(self, engine, version, with_data=False): - """migrate up to a new version of the db. - - We allow for data insertion and post checks at every - migration version with special _pre_upgrade_### and - _check_### functions in the main test. - """ - if with_data: - data = None - pre_upgrade = getattr(self, "_pre_upgrade_%3.3d" % version, None) - if pre_upgrade: - data = pre_upgrade(engine) - - migration_api.upgrade(engine, - TestMigrations.REPOSITORY, - version) - self.assertEqual(version, - migration_api.db_version(engine, - TestMigrations.REPOSITORY)) - - if with_data: - check = getattr(self, "_check_%3.3d" % version, None) - if check: - check(engine, data) + self._walk_versions(True, False) def _create_unversioned_001_db(self, engine): # Create the initial version of the images table @@ -432,7 +105,8 @@ class TestMigrations(test_utils.BaseTestCase): sqlalchemy.Column('deleted', sqlalchemy.Boolean(), nullable=False, - default=False)) + default=False), + mysql_engine='InnoDB') images_001.create() def test_version_control_existing_db(self): @@ -441,25 +115,30 @@ class TestMigrations(test_utils.BaseTestCase): under version control and checks that it can be upgraded without errors. """ - for key, engine in self.engines.items(): - self._create_unversioned_001_db(engine) - self._walk_versions(engine, self.snake_walk, initial_version=1) + self._create_unversioned_001_db(self.migrate_engine) + + old_version = migration.INIT_VERSION + #we must start from version 1 + migration.INIT_VERSION = 1 + self.addCleanup(setattr, migration, 'INIT_VERSION', old_version) + + self._walk_versions(False, False) def _pre_upgrade_003(self, engine): now = datetime.datetime.now() - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') data = {'deleted': False, 'created_at': now, 'updated_at': now, 'type': 'kernel', 'status': 'active', 'is_public': True} images.insert().values(data).execute() return data def _check_003(self, engine, data): - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') self.assertTrue('type' not in images.c, "'type' column found in images table columns! " "images table columns reported by metadata: %s\n" % images.c.keys()) - images_prop = get_table(engine, 'image_properties') + images_prop = db_utils.get_table(engine, 'image_properties') result = images_prop.select().execute() types = [] for row in result: @@ -472,7 +151,7 @@ class TestMigrations(test_utils.BaseTestCase): data. """ now = timeutils.utcnow() - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') data = [ { 'deleted': False, 'created_at': now, 'updated_at': now, @@ -484,13 +163,13 @@ class TestMigrations(test_utils.BaseTestCase): def _check_004(self, engine, data): """Assure that checksum data is present on table""" - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') self.assertIn('checksum', images.c) self.assertEqual(images.c['checksum'].type.length, 32) def _pre_upgrade_005(self, engine): now = timeutils.utcnow() - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') data = [ { 'deleted': False, 'created_at': now, 'updated_at': now, @@ -504,7 +183,7 @@ class TestMigrations(test_utils.BaseTestCase): def _check_005(self, engine, data): - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') select = images.select().execute() sizes = [row['size'] for row in select if row['size'] is not None] @@ -515,7 +194,7 @@ class TestMigrations(test_utils.BaseTestCase): def _pre_upgrade_006(self, engine): now = timeutils.utcnow() - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') image_data = [ { 'deleted': False, 'created_at': now, 'updated_at': now, @@ -525,7 +204,7 @@ class TestMigrations(test_utils.BaseTestCase): ] engine.execute(images.insert(), image_data) - images_properties = get_table(engine, 'image_properties') + images_properties = db_utils.get_table(engine, 'image_properties') properties_data = [ { 'id': 10, 'image_id': 9999, 'updated_at': now, @@ -536,7 +215,7 @@ class TestMigrations(test_utils.BaseTestCase): return properties_data def _check_006(self, engine, data): - images_properties = get_table(engine, 'image_properties') + images_properties = db_utils.get_table(engine, 'image_properties') select = images_properties.select().execute() # load names from name collumn @@ -562,7 +241,7 @@ class TestMigrations(test_utils.BaseTestCase): None), ] - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') for created_at, updated_at in initial_values: row = dict(deleted=False, created_at=created_at, @@ -578,7 +257,7 @@ class TestMigrations(test_utils.BaseTestCase): def _check_010(self, engine, data): values = dict((c, u) for c, u in data) - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') for row in images.select().execute(): if row['created_at'] in values: # updated_at should be unchanged if not previous NULL, or @@ -595,9 +274,9 @@ class TestMigrations(test_utils.BaseTestCase): image_properties gets updated to point to new UUID keys. """ - images = get_table(engine, 'images') - image_members = get_table(engine, 'image_members') - image_properties = get_table(engine, 'image_properties') + images = db_utils.get_table(engine, 'images') + image_members = db_utils.get_table(engine, 'image_members') + image_properties = db_utils.get_table(engine, 'image_properties') # Insert kernel, ramdisk and normal images now = timeutils.utcnow() @@ -631,9 +310,9 @@ class TestMigrations(test_utils.BaseTestCase): return test_data def _check_012(self, engine, test_data): - images = get_table(engine, 'images') - image_members = get_table(engine, 'image_members') - image_properties = get_table(engine, 'image_properties') + images = db_utils.get_table(engine, 'images') + image_members = db_utils.get_table(engine, 'image_members') + image_properties = db_utils.get_table(engine, 'image_properties') # Find kernel, ramdisk and normal images. Make sure id has been # changed to a uuid @@ -675,9 +354,9 @@ class TestMigrations(test_utils.BaseTestCase): self.assertEqual(row['value'], uuids['ramdisk']) def _post_downgrade_012(self, engine): - images = get_table(engine, 'images') - image_members = get_table(engine, 'image_members') - image_properties = get_table(engine, 'image_properties') + images = db_utils.get_table(engine, 'images') + image_members = db_utils.get_table(engine, 'image_members') + image_properties = db_utils.get_table(engine, 'image_properties') # Find kernel, ramdisk and normal images. Make sure id has been # changed back to an integer @@ -784,7 +463,7 @@ class TestMigrations(test_utils.BaseTestCase): self._assert_invalid_swift_uri_raises_bad_store_uri(legacy_parse_uri) def _pre_upgrade_015(self, engine): - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') unquoted_locations = [ 'swift://acct:usr:pass@example.com/container/obj-id', 'file://foo', @@ -805,7 +484,7 @@ class TestMigrations(test_utils.BaseTestCase): return data def _check_015(self, engine, data): - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') quoted_locations = [ 'swift://acct%3Ausr:pass@example.com/container/obj-id', 'file://foo', @@ -816,7 +495,7 @@ class TestMigrations(test_utils.BaseTestCase): self.assertIn(loc, locations) def _pre_upgrade_016(self, engine): - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') now = datetime.datetime.now() temp = dict(deleted=False, created_at=now, @@ -827,7 +506,7 @@ class TestMigrations(test_utils.BaseTestCase): min_ram=0, id='fake-image-id1') images.insert().values(temp).execute() - image_members = get_table(engine, 'image_members') + image_members = db_utils.get_table(engine, 'image_members') now = datetime.datetime.now() data = {'deleted': False, 'created_at': now, @@ -839,7 +518,7 @@ class TestMigrations(test_utils.BaseTestCase): return data def _check_016(self, engine, data): - image_members = get_table(engine, 'image_members') + image_members = db_utils.get_table(engine, 'image_members') self.assertTrue('status' in image_members.c, "'status' column found in image_members table " "columns! image_members table columns: %s" @@ -847,8 +526,8 @@ class TestMigrations(test_utils.BaseTestCase): def test_legacy_parse_swift_uri_017(self): metadata_encryption_key = 'a' * 16 - self.config(metadata_encryption_key=metadata_encryption_key) - + CONF.set_override('metadata_encryption_key', metadata_encryption_key) + self.addCleanup(CONF.reset) (legacy_parse_uri, encrypt_location) = from_migration_import( '017_quote_encrypted_swift_credentials', ['legacy_parse_uri', 'encrypt_location']) @@ -862,8 +541,9 @@ class TestMigrations(test_utils.BaseTestCase): def _pre_upgrade_017(self, engine): metadata_encryption_key = 'a' * 16 - self.config(metadata_encryption_key=metadata_encryption_key) - images = get_table(engine, 'images') + CONF.set_override('metadata_encryption_key', metadata_encryption_key) + self.addCleanup(CONF.reset) + images = db_utils.get_table(engine, 'images') unquoted = 'swift://acct:usr:pass@example.com/container/obj-id' encrypted_unquoted = crypt.urlsafe_encrypt( metadata_encryption_key, @@ -904,7 +584,7 @@ class TestMigrations(test_utils.BaseTestCase): def _check_017(self, engine, data): metadata_encryption_key = 'a' * 16 quoted = 'swift://acct%3Ausr:pass@example.com/container/obj-id' - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') result = images.select().execute() locations = map(lambda x: x['location'], result) actual_location = [] @@ -929,7 +609,7 @@ class TestMigrations(test_utils.BaseTestCase): self.fail(_("location: %s data lost") % location) def _pre_upgrade_019(self, engine): - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') now = datetime.datetime.now() base_values = { 'deleted': False, @@ -952,24 +632,24 @@ class TestMigrations(test_utils.BaseTestCase): return data def _check_019(self, engine, data): - image_locations = get_table(engine, 'image_locations') + image_locations = db_utils.get_table(engine, 'image_locations') records = image_locations.select().execute().fetchall() locations = dict([(il.image_id, il.value) for il in records]) self.assertEqual(locations.get('fake-19-1'), 'http://glance.example.com') def _check_020(self, engine, data): - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') self.assertNotIn('location', images.c) def _pre_upgrade_026(self, engine): - image_locations = get_table(engine, 'image_locations') + image_locations = db_utils.get_table(engine, 'image_locations') now = datetime.datetime.now() image_id = 'fake_id' url = 'file:///some/place/onthe/fs' - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') temp = dict(deleted=False, created_at=now, updated_at=now, @@ -989,7 +669,7 @@ class TestMigrations(test_utils.BaseTestCase): return image_id def _check_026(self, engine, data): - image_locations = get_table(engine, 'image_locations') + image_locations = db_utils.get_table(engine, 'image_locations') results = image_locations.select()\ .where(image_locations.c.image_id == data).execute() @@ -1019,7 +699,7 @@ class TestMigrations(test_utils.BaseTestCase): owner_index = "owner_image_idx" columns = ["owner"] - images_table = get_table(engine, 'images') + images_table = db_utils.get_table(engine, 'images') index_data = [(idx.name, idx.columns.keys()) for idx in images_table.indexes @@ -1031,7 +711,7 @@ class TestMigrations(test_utils.BaseTestCase): owner_index = "owner_image_idx" columns = ["owner"] - images_table = get_table(engine, 'images') + images_table = db_utils.get_table(engine, 'images') index_data = [(idx.name, idx.columns.keys()) for idx in images_table.indexes @@ -1040,7 +720,7 @@ class TestMigrations(test_utils.BaseTestCase): self.assertNotIn((owner_index, columns), index_data) def _pre_upgrade_029(self, engine): - image_locations = get_table(engine, 'image_locations') + image_locations = db_utils.get_table(engine, 'image_locations') meta_data = {'somelist': ['a', 'b', 'c'], 'avalue': 'hello', 'adict': {}} @@ -1049,7 +729,7 @@ class TestMigrations(test_utils.BaseTestCase): image_id = 'fake_029_id' url = 'file:///some/place/onthe/fs029' - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') temp = dict(deleted=False, created_at=now, updated_at=now, @@ -1074,7 +754,7 @@ class TestMigrations(test_utils.BaseTestCase): def _check_029(self, engine, data): meta_data = data[0] image_id = data[1] - image_locations = get_table(engine, 'image_locations') + image_locations = db_utils.get_table(engine, 'image_locations') records = image_locations.select().\ where(image_locations.c.image_id == image_id).execute().fetchall() @@ -1086,7 +766,7 @@ class TestMigrations(test_utils.BaseTestCase): def _post_downgrade_029(self, engine): image_id = 'fake_029_id' - image_locations = get_table(engine, 'image_locations') + image_locations = db_utils.get_table(engine, 'image_locations') records = image_locations.select().\ where(image_locations.c.image_id == image_id).execute().fetchall() @@ -1139,10 +819,10 @@ class TestMigrations(test_utils.BaseTestCase): def _post_downgrade_030(self, engine): self.assertRaises(sqlalchemy.exc.NoSuchTableError, - get_table, engine, 'tasks') + db_utils.get_table, engine, 'tasks') def _pre_upgrade_031(self, engine): - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') now = datetime.datetime.now() image_id = 'fake_031_id' temp = dict(deleted=False, @@ -1155,7 +835,7 @@ class TestMigrations(test_utils.BaseTestCase): id=image_id) images.insert().values(temp).execute() - locations_table = get_table(engine, 'image_locations') + locations_table = db_utils.get_table(engine, 'image_locations') locations = [ ('file://ab', '{"a": "yo yo"}'), ('file://ab', '{}'), @@ -1174,7 +854,7 @@ class TestMigrations(test_utils.BaseTestCase): return image_id def _check_031(self, engine, image_id): - locations_table = get_table(engine, 'image_locations') + locations_table = db_utils.get_table(engine, 'image_locations') result = locations_table.select()\ .where(locations_table.c.image_id == image_id)\ .execute().fetchall() @@ -1189,9 +869,9 @@ class TestMigrations(test_utils.BaseTestCase): def _pre_upgrade_032(self, engine): self.assertRaises(sqlalchemy.exc.NoSuchTableError, - get_table, engine, 'task_info') + db_utils.get_table, engine, 'task_info') - tasks = get_table(engine, 'tasks') + tasks = db_utils.get_table(engine, 'tasks') now = datetime.datetime.now() base_values = { 'deleted': False, @@ -1221,7 +901,7 @@ class TestMigrations(test_utils.BaseTestCase): return data def _check_032(self, engine, data): - task_info_table = get_table(engine, 'task_info') + task_info_table = db_utils.get_table(engine, 'task_info') task_info_refs = task_info_table.select().execute().fetchall() @@ -1233,16 +913,16 @@ class TestMigrations(test_utils.BaseTestCase): self.assertEqual(task_info_refs[x].result, data[x]['result']) self.assertIsNone(task_info_refs[x].message) - tasks_table = get_table(engine, 'tasks') + tasks_table = db_utils.get_table(engine, 'tasks') self.assertNotIn('input', tasks_table.c) self.assertNotIn('result', tasks_table.c) self.assertNotIn('message', tasks_table.c) def _post_downgrade_032(self, engine): self.assertRaises(sqlalchemy.exc.NoSuchTableError, - get_table, engine, 'task_info') + db_utils.get_table, engine, 'task_info') - tasks_table = get_table(engine, 'tasks') + tasks_table = db_utils.get_table(engine, 'tasks') records = tasks_table.select().execute().fetchall() self.assertEqual(len(records), 2) @@ -1259,8 +939,8 @@ class TestMigrations(test_utils.BaseTestCase): self.assertIsNone(task_2.message) def _pre_upgrade_033(self, engine): - images = get_table(engine, 'images') - image_locations = get_table(engine, 'image_locations') + images = db_utils.get_table(engine, 'images') + image_locations = db_utils.get_table(engine, 'image_locations') now = datetime.datetime.now() image_id = 'fake_id_028_%d' @@ -1291,7 +971,7 @@ class TestMigrations(test_utils.BaseTestCase): return image_id_list def _check_033(self, engine, data): - image_locations = get_table(engine, 'image_locations') + image_locations = db_utils.get_table(engine, 'image_locations') self.assertIn('status', image_locations.c) self.assertEqual(image_locations.c['status'].type.length, 30) @@ -1308,11 +988,11 @@ class TestMigrations(test_utils.BaseTestCase): self.assertEqual(r[0]['status'], status_list[idx]) def _post_downgrade_033(self, engine): - image_locations = get_table(engine, 'image_locations') + image_locations = db_utils.get_table(engine, 'image_locations') self.assertNotIn('status', image_locations.c) def _pre_upgrade_034(self, engine): - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') now = datetime.datetime.now() image_id = 'fake_id_034' @@ -1327,7 +1007,7 @@ class TestMigrations(test_utils.BaseTestCase): images.insert().values(temp).execute() def _check_034(self, engine, data): - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') self.assertIn('virtual_size', images.c) result = (images.select() @@ -1336,20 +1016,20 @@ class TestMigrations(test_utils.BaseTestCase): self.assertIsNone(result.virtual_size) def _post_downgrade_034(self, engine): - images = get_table(engine, 'images') + images = db_utils.get_table(engine, 'images') self.assertNotIn('virtual_size', images.c) def _pre_upgrade_035(self, engine): self.assertRaises(sqlalchemy.exc.NoSuchTableError, - get_table, engine, 'metadef_namespaces') + db_utils.get_table, engine, 'metadef_namespaces') self.assertRaises(sqlalchemy.exc.NoSuchTableError, - get_table, engine, 'metadef_properties') + db_utils.get_table, engine, 'metadef_properties') self.assertRaises(sqlalchemy.exc.NoSuchTableError, - get_table, engine, 'metadef_objects') + db_utils.get_table, engine, 'metadef_objects') self.assertRaises(sqlalchemy.exc.NoSuchTableError, - get_table, engine, 'metadef_resource_types') + db_utils.get_table, engine, 'metadef_resource_types') self.assertRaises(sqlalchemy.exc.NoSuchTableError, - get_table, engine, + db_utils.get_table, engine, 'metadef_namespace_resource_types') def _check_035(self, engine, data): @@ -1450,13 +1130,48 @@ class TestMigrations(test_utils.BaseTestCase): def _post_downgrade_035(self, engine): self.assertRaises(sqlalchemy.exc.NoSuchTableError, - get_table, engine, 'metadef_namespaces') + db_utils.get_table, engine, 'metadef_namespaces') self.assertRaises(sqlalchemy.exc.NoSuchTableError, - get_table, engine, 'metadef_properties') + db_utils.get_table, engine, 'metadef_properties') self.assertRaises(sqlalchemy.exc.NoSuchTableError, - get_table, engine, 'metadef_objects') + db_utils.get_table, engine, 'metadef_objects') self.assertRaises(sqlalchemy.exc.NoSuchTableError, - get_table, engine, 'metadef_resource_types') + db_utils.get_table, engine, 'metadef_resource_types') self.assertRaises(sqlalchemy.exc.NoSuchTableError, - get_table, engine, + db_utils.get_table, engine, 'metadef_namespace_resource_types') + + +class TestMysqlMigrations(test_base.MySQLOpportunisticTestCase, + MigrationsMixin): + + def test_mysql_innodb_tables(self): + migration.db_sync(engine=self.migrate_engine) + + total = self.migrate_engine.execute( + "SELECT COUNT(*) " + "FROM information_schema.TABLES " + "WHERE TABLE_SCHEMA='%s'" + % self.migrate_engine.url.database) + self.assertTrue(total.scalar() > 0, "No tables found. Wrong schema?") + + noninnodb = self.migrate_engine.execute( + "SELECT count(*) " + "FROM information_schema.TABLES " + "WHERE TABLE_SCHEMA='%s' " + "AND ENGINE!='InnoDB' " + "AND TABLE_NAME!='migrate_version'" + % self.migrate_engine.url.database) + count = noninnodb.scalar() + self.assertEqual(count, 0, "%d non InnoDB tables created" % count) + + +class TestPostgresqlMigrations(test_base.PostgreSQLOpportunisticTestCase, + MigrationsMixin): + pass + + +class TestSqliteMigrations(test_base.DbTestCase, + MigrationsMixin): + def test_walk_versions(self): + self._walk_versions(True, True) diff --git a/test-requirements.txt b/test-requirements.txt index cb34add1f0..e75c3775aa 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,7 +19,7 @@ requests>=1.2.1,!=2.4.0 testrepository>=0.0.18 testtools>=0.9.34 psutil>=1.1.1,<2.0.0 - +oslotest>=1.1.0 # Optional packages that should be installed when testing MySQL-python psycopg2