openstack/cinder
Code Issues Proposed changes

db: Integrate alembic

Browse Source 
Start the move to alembic for real by integrating it into the 'db sync'
command flow. sqlalchemy-migrate is not removed entirely. Not yet
anyway. Instead, we will apply all sqlalchemy-migrate migrations up to
the final '140_create_project_default_volume_type' migration, after
which we will switch over to alembic. In a future release we can remove
the sqlalchemy-migrate migrations and rely entirely on alembic.

Change-Id: Id30929de443d5f7e3923d1061a9f7bbe8b949d5a
Signed-off-by: Stephen Finucane <stephenfin@redhat.com>
This commit is contained in:
Stephen Finucane 2020-11-02 09:53:14 +00:00
parent 069b4c4775
commit ee771bbdbc
5 changed files with 422 additions and 353 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

215
cinder/db/migration.py
View File

@ -18,135 +18,152 @@
import os import os
from alembic import command as alembic_api
from alembic import config as alembic_config
from alembic import migration as alembic_migration
from migrate import exceptions as migrate_exceptions from migrate import exceptions as migrate_exceptions
from migrate.versioning import api as migrate_api from migrate.versioning import api as migrate_api
from migrate.versioning import repository as migrate_repository from migrate.versioning import repository as migrate_repo
from oslo_config import cfg from oslo_config import cfg
from oslo_db import exception
from oslo_db import options from oslo_db import options
import sqlalchemy as sa from oslo_log import log as logging
from cinder.db.sqlalchemy import api as db_api from cinder.db.sqlalchemy import api as db_api
from cinder.i18n import _
options.set_defaults(cfg.CONF) options.set_defaults(cfg.CONF)
INIT_VERSION = 134 LOG = logging.getLogger(__name__)
LEGACY_MIGRATIONS_PATH = os.path.join(
os.path.abspath(os.path.dirname(__file__)), MIGRATE_INIT_VERSION = 134
'legacy_migrations', MIGRATE_MIGRATIONS_PATH = ALEMBIC_INIT_VERSION = '921e1a36b076'
)
def _find_migrate_repo(abs_path): def _find_migrate_repo():
"""Get the project's change script repository """Get the project's change script repository
:param abs_path: Absolute path to migrate repository :returns: An instance of ``migrate.versioning.repository.Repository``
""" """
if not os.path.exists(abs_path): path = os.path.join(
raise exception.DBMigrationError("Path %s not found" % abs_path) os.path.abspath(os.path.dirname(__file__)), 'legacy_migrations',
return migrate_repository.Repository(abs_path)
def _migrate_db_version_control(engine, abs_path, version=None):
"""Mark a database as under this repository's version control.
Once a database is under version control, schema changes should
only be done via change scripts in this repository.
:param engine: SQLAlchemy engine instance for a given database
:param abs_path: Absolute path to migrate repository
:param version: Initial database version
"""
repository = _find_migrate_repo(abs_path)
try:
migrate_api.version_control(engine, repository, version)
except migrate_exceptions.InvalidVersionError as ex:
raise exception.DBMigrationError("Invalid version : %s" % ex)
except migrate_exceptions.DatabaseAlreadyControlledError:
raise exception.DBMigrationError("Database is already controlled.")
return version
def _migrate_db_version(engine, abs_path, init_version):
"""Show the current version of the repository.
:param engine: SQLAlchemy engine instance for a given database
:param abs_path: Absolute path to migrate repository
:param init_version: Initial database version
"""
repository = _find_migrate_repo(abs_path)
try:
return migrate_api.db_version(engine, repository)
except migrate_exceptions.DatabaseNotControlledError:
pass
meta = sa.MetaData()
meta.reflect(bind=engine)
tables = meta.tables
if (
len(tables) == 0 or
'alembic_version' in tables or
'migrate_version' in tables
):
_migrate_db_version_control(engine, abs_path, version=init_version)
return migrate_api.db_version(engine, repository)
msg = _(
"The database is not under version control, but has tables. "
"Please stamp the current version of the schema manually."
) )
raise exception.DBMigrationError(msg) return migrate_repo.Repository(path)
def _migrate_db_sync(engine, abs_path, version=None, init_version=0): def _find_alembic_conf():
"""Upgrade or downgrade a database. """Get the project's alembic configuration
Function runs the upgrade() or downgrade() functions in change scripts. :returns: An instance of ``alembic.config.Config``
:param engine: SQLAlchemy engine instance for a given database
:param abs_path: Absolute path to migrate repository.
:param version: Database will upgrade/downgrade until this version.
If None - database will update to the latest available version.
:param init_version: Initial database version
""" """
path = os.path.join(
os.path.abspath(os.path.dirname(__file__)), 'alembic.ini')
if version is not None: config = alembic_config.Config(os.path.abspath(path))
try: # we don't want to use the logger configuration from the file, which is
version = int(version) # only really intended for the CLI
except ValueError: # https://stackoverflow.com/a/42691781/613428
raise exception.DBMigrationError(_("version should be an integer")) config.attributes['configure_logger'] = False
current_version = _migrate_db_version(engine, abs_path, init_version) return config
repository = _find_migrate_repo(abs_path)
if version is None or version > current_version:
try: def _is_database_under_migrate_control(engine, repository):
return migrate_api.upgrade(engine, repository, version) try:
except Exception as ex: migrate_api.db_version(engine, repository)
raise exception.DBMigrationError(ex) return True
else: except migrate_exceptions.DatabaseNotControlledError:
return migrate_api.downgrade(engine, repository, version) return False
def _is_database_under_alembic_control(engine):
with engine.connect() as conn:
context = alembic_migration.MigrationContext.configure(conn)
return bool(context.get_current_revision())
def _init_alembic_on_legacy_database(engine, repository, config):
"""Init alembic in an existing environment with sqlalchemy-migrate."""
LOG.info(
'The database is still under sqlalchemy-migrate control; '
'applying any remaining sqlalchemy-migrate-based migrations '
'and fake applying the initial alembic migration'
)
migrate_api.upgrade(engine, repository)
# re-use the connection rather than creating a new one
with engine.begin() as connection:
config.attributes['connection'] = connection
alembic_api.stamp(config, ALEMBIC_INIT_VERSION)
def _upgrade_alembic(engine, config, version):
# re-use the connection rather than creating a new one
with engine.begin() as connection:
config.attributes['connection'] = connection
alembic_api.upgrade(config, version or 'head')
def db_version(): def db_version():
"""Get database version.""" """Get database version."""
return _migrate_db_version( repository = _find_migrate_repo()
db_api.get_engine(), engine = db_api.get_engine()
LEGACY_MIGRATIONS_PATH,
INIT_VERSION) migrate_version = None
if _is_database_under_migrate_control(engine, repository):
migrate_version = migrate_api.db_version(engine, repository)
alembic_version = None
if _is_database_under_alembic_control(engine):
alembic_version = alembic_api.current(engine)
return alembic_version or migrate_version
def db_sync(version=None, engine=None): def db_sync(version=None, engine=None):
"""Migrate the database to `version` or the most recent version.""" """Migrate the database to `version` or the most recent version.
We're currently straddling two migration systems, sqlalchemy-migrate and
alembic. This handles both by ensuring we switch from one to the other at
the appropriate moment.
"""
# if the user requested a specific version, check if it's an integer: if
# so, we're almost certainly in sqlalchemy-migrate land and won't support
# that
if version is not None and version.isdigit():
raise ValueError(
'You requested an sqlalchemy-migrate database version; this is '
'no longer supported'
)
if engine is None: if engine is None:
engine = db_api.get_engine() engine = db_api.get_engine()
return _migrate_db_sync( repository = _find_migrate_repo()
engine=engine, config = _find_alembic_conf()
abs_path=LEGACY_MIGRATIONS_PATH,
version=version, # discard the URL encoded in alembic.ini in favour of the URL configured
init_version=INIT_VERSION) # for the engine by the database fixtures, casting from
# 'sqlalchemy.engine.url.URL' to str in the process. This returns a
# RFC-1738 quoted URL, which means that a password like "foo@" will be
# turned into "foo%40". This in turns causes a problem for
# set_main_option() because that uses ConfigParser.set, which (by design)
# uses *python* interpolation to write the string out ... where "%" is the
# special python interpolation character! Avoid this mismatch by quoting
# all %'s for the set below.
engine_url = str(engine.url).replace('%', '%%')
config.set_main_option('sqlalchemy.url', str(engine_url))
# if we're in a deployment where sqlalchemy-migrate is already present,
# then apply all the updates for that and fake apply the initial alembic
# migration; if we're not then 'upgrade' will take care of everything
# this should be a one-time operation
if (
_is_database_under_migrate_control(engine, repository) and
not _is_database_under_alembic_control(engine)
):
_init_alembic_on_legacy_database(engine, repository, config)
# apply anything later
LOG.info('Applying migration(s)')
_upgrade_alembic(engine, config, version)
LOG.info('Migration(s) applied')

28
cinder/db/migrations/env.py
View File

@ -20,9 +20,10 @@ from sqlalchemy import pool
# access to the values within the .ini file in use. # access to the values within the .ini file in use.
config = context.config config = context.config
# Interpret the config file for Python logging. # Interpret the config file for Python logging unless we're told not to.
# This line sets up loggers basically. # This line sets up loggers basically.
fileConfig(config.config_file_name) if config.attributes.get('configure_logger', True):
fileConfig(config.config_file_name)
# add your model's MetaData object here # add your model's MetaData object here
# for 'autogenerate' support # for 'autogenerate' support
@ -57,12 +58,25 @@ def run_migrations_online():
In this scenario we need to create an Engine and associate a connection In this scenario we need to create an Engine and associate a connection
with the context. with the context.
This is modified from the default based on the below, since we want to
share an engine when unit testing so in-memory database testing actually
works.
https://alembic.sqlalchemy.org/en/latest/cookbook.html#connection-sharing
""" """
connectable = engine_from_config( connectable = config.attributes.get('connection', None)
config.get_section(config.config_ini_section),
prefix="sqlalchemy.", if connectable is None:
poolclass=pool.NullPool, # only create Engine if we don't have a Connection from the outside
) connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
# when connectable is already a Connection object, calling connect() gives
# us a *branched connection*
with connectable.connect() as connection: with connectable.connect() as connection:
context.configure( context.configure(

360
cinder/tests/unit/db/test_migration.py
View File

@ -10,243 +10,177 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import os
import tempfile
from unittest import mock from unittest import mock
from migrate import exceptions as migrate_exception from alembic import command as alembic_api
from alembic.runtime import migration as alembic_migration
from migrate import exceptions as migrate_exceptions
from migrate.versioning import api as migrate_api from migrate.versioning import api as migrate_api
from migrate.versioning import repository as migrate_repository
from oslo_db import exception as db_exception
from oslo_db.sqlalchemy import enginefacade
from oslo_db.sqlalchemy import test_fixtures as db_fixtures
from oslotest import base as test_base from oslotest import base as test_base
import sqlalchemy
from cinder.db import migration from cinder.db import migration
from cinder import utils from cinder.db.sqlalchemy import api as db_api
class TestMigrationCommon( class TestDBSync(test_base.BaseTestCase):
db_fixtures.OpportunisticDBTestMixin, test_base.BaseTestCase,
):
def setUp(self): def test_db_sync_legacy_version(self):
super().setUp() """We don't allow users to request legacy versions."""
self.assertRaises(ValueError, migration.db_sync, '402')
self.engine = enginefacade.writer.get_engine() @mock.patch.object(migration, '_upgrade_alembic')
@mock.patch.object(migration, '_init_alembic_on_legacy_database')
self.path = tempfile.mkdtemp('test_migration') @mock.patch.object(migration, '_is_database_under_alembic_control')
self.path1 = tempfile.mkdtemp('test_migration') @mock.patch.object(migration, '_is_database_under_migrate_control')
self.return_value = '/home/openstack/migrations' @mock.patch.object(migration, '_find_alembic_conf')
self.return_value1 = '/home/extension/migrations' @mock.patch.object(migration, '_find_migrate_repo')
self.init_version = 1 @mock.patch.object(db_api, 'get_engine')
self.test_version = 123 def _test_db_sync(
self, has_migrate, has_alembic, mock_get_engine, mock_find_repo,
self.patcher_repo = mock.patch.object(migrate_repository, 'Repository') mock_find_conf, mock_is_migrate, mock_is_alembic, mock_init,
self.repository = self.patcher_repo.start() mock_upgrade,
self.repository.side_effect = [self.return_value, self.return_value1] ):
mock_is_migrate.return_value = has_migrate
self.mock_api_db = mock.patch.object(migrate_api, 'db_version') mock_is_alembic.return_value = has_alembic
self.mock_api_db_version = self.mock_api_db.start() migration.db_sync()
self.mock_api_db_version.return_value = self.test_version mock_get_engine.assert_called_once_with()
mock_find_repo.assert_called_once_with()
def tearDown(self): mock_find_conf.assert_called_once_with()
os.rmdir(self.path) mock_find_conf.return_value.set_main_option.assert_called_once_with(
self.mock_api_db.stop() 'sqlalchemy.url', str(mock_get_engine.return_value.url),
self.patcher_repo.stop()
super().tearDown()
def test_find_migrate_repo_path_not_found(self):
self.assertRaises(
db_exception.DBMigrationError,
migration._find_migrate_repo,
"/foo/bar/",
) )
mock_is_migrate.assert_called_once_with(
mock_get_engine.return_value, mock_find_repo.return_value)
if has_migrate:
mock_is_alembic.assert_called_once_with(
mock_get_engine.return_value)
else:
mock_is_alembic.assert_not_called()
def test_find_migrate_repo_called_once(self): # we should only attempt the upgrade of the remaining
my_repository = migration._find_migrate_repo(self.path) # sqlalchemy-migrate-based migrations and fake apply of the initial
self.repository.assert_called_once_with(self.path) # alembic migrations if sqlalchemy-migrate is in place but alembic
self.assertEqual(self.return_value, my_repository) # hasn't been used yet
if has_migrate and not has_alembic:
mock_init.assert_called_once_with(
mock_get_engine.return_value,
mock_find_repo.return_value, mock_find_conf.return_value)
else:
mock_init.assert_not_called()
def test_find_migrate_repo_called_few_times(self): # however, we should always attempt to upgrade the requested migration
repo1 = migration._find_migrate_repo(self.path) # to alembic
repo2 = migration._find_migrate_repo(self.path1) mock_upgrade.assert_called_once_with(
self.assertNotEqual(repo1, repo2) mock_get_engine.return_value, mock_find_conf.return_value, None)
def test_db_version_control(self): def test_db_sync_new_deployment(self):
with utils.nested_contexts( """Mimic a new deployment without existing sqlalchemy-migrate cruft."""
mock.patch.object(migration, '_find_migrate_repo'), has_migrate = False
mock.patch.object(migrate_api, 'version_control'), has_alembic = False
) as (mock_find_repo, mock_version_control): self._test_db_sync(has_migrate, has_alembic)
mock_find_repo.return_value = self.return_value
version = migration._migrate_db_version_control( def test_db_sync_with_existing_migrate_database(self):
self.engine, self.path, self.test_version) """Mimic a deployment currently managed by sqlalchemy-migrate."""
has_migrate = True
has_alembic = False
self._test_db_sync(has_migrate, has_alembic)
self.assertEqual(self.test_version, version) def test_db_sync_with_existing_alembic_database(self):
mock_version_control.assert_called_once_with( """Mimic a deployment that's already switched to alembic."""
self.engine, self.return_value, self.test_version) has_migrate = True
has_alembic = True
self._test_db_sync(has_migrate, has_alembic)
@mock.patch.object(migration, '_find_migrate_repo')
@mock.patch.object(migrate_api, 'version_control') @mock.patch.object(alembic_api, 'current')
def test_db_version_control_version_less_than_actual_version( @mock.patch.object(migrate_api, 'db_version')
self, mock_version_control, mock_find_repo, @mock.patch.object(migration, '_is_database_under_alembic_control')
@mock.patch.object(migration, '_is_database_under_migrate_control')
@mock.patch.object(db_api, 'get_engine')
@mock.patch.object(migration, '_find_migrate_repo')
class TestDBVersion(test_base.BaseTestCase):
def test_db_version_migrate(
self, mock_find_repo, mock_get_engine, mock_is_migrate,
mock_is_alembic, mock_migrate_version, mock_alembic_version,
): ):
mock_find_repo.return_value = self.return_value """Database is controlled by sqlalchemy-migrate."""
mock_version_control.side_effect = \ mock_is_migrate.return_value = True
migrate_exception.DatabaseAlreadyControlledError mock_is_alembic.return_value = False
self.assertRaises( ret = migration.db_version()
db_exception.DBMigrationError, self.assertEqual(mock_migrate_version.return_value, ret)
migration._migrate_db_version_control, self.engine, mock_find_repo.assert_called_once_with()
self.path, self.test_version - 1) mock_get_engine.assert_called_once_with()
mock_is_migrate.assert_called_once()
mock_is_alembic.assert_called_once()
mock_migrate_version.assert_called_once_with(
mock_get_engine.return_value, mock_find_repo.return_value)
mock_alembic_version.assert_not_called()
@mock.patch.object(migration, '_find_migrate_repo') def test_db_version_alembic(
@mock.patch.object(migrate_api, 'version_control') self, mock_find_repo, mock_get_engine, mock_is_migrate,
def test_db_version_control_version_greater_than_actual_version( mock_is_alembic, mock_migrate_version, mock_alembic_version,
self, mock_version_control, mock_find_repo,
): ):
mock_find_repo.return_value = self.return_value """Database is controlled by alembic."""
mock_version_control.side_effect = \ mock_is_migrate.return_value = False
migrate_exception.InvalidVersionError mock_is_alembic.return_value = True
self.assertRaises( ret = migration.db_version()
db_exception.DBMigrationError, self.assertEqual(mock_alembic_version.return_value, ret)
migration._migrate_db_version_control, self.engine, mock_find_repo.assert_called_once_with()
self.path, self.test_version + 1) mock_get_engine.assert_called_once_with()
mock_is_migrate.assert_called_once()
mock_is_alembic.assert_called_once()
mock_migrate_version.assert_not_called()
mock_alembic_version.assert_called_once_with(
mock_get_engine.return_value)
def test_db_version_return(self): def test_db_version_not_controlled(
ret_val = migration._migrate_db_version( self, mock_find_repo, mock_get_engine, mock_is_migrate,
self.engine, self.path, self.init_version) mock_is_alembic, mock_migrate_version, mock_alembic_version,
self.assertEqual(self.test_version, ret_val) ):
"""Database is not controlled."""
mock_is_migrate.return_value = False
mock_is_alembic.return_value = False
ret = migration.db_version()
self.assertIsNone(ret)
mock_find_repo.assert_called_once_with()
mock_get_engine.assert_called_once_with()
mock_is_migrate.assert_called_once()
mock_is_alembic.assert_called_once()
mock_migrate_version.assert_not_called()
mock_alembic_version.assert_not_called()
def test_db_version_raise_not_controlled_error_first(self):
with mock.patch.object(
migration, '_migrate_db_version_control',
) as mock_ver:
self.mock_api_db_version.side_effect = [
migrate_exception.DatabaseNotControlledError('oups'),
self.test_version]
ret_val = migration._migrate_db_version( class TestDatabaseUnderVersionControl(test_base.BaseTestCase):
self.engine, self.path, self.init_version)
self.assertEqual(self.test_version, ret_val)
mock_ver.assert_called_once_with(
self.engine, self.path, version=self.init_version)
def test_db_version_raise_not_controlled_error_tables(self): @mock.patch.object(migrate_api, 'db_version')
with mock.patch.object(sqlalchemy, 'MetaData') as mock_meta: def test__is_database_under_migrate_control__true(self, mock_db_version):
self.mock_api_db_version.side_effect = \ ret = migration._is_database_under_migrate_control('engine', 'repo')
migrate_exception.DatabaseNotControlledError('oups') self.assertTrue(ret)
my_meta = mock.MagicMock() mock_db_version.assert_called_once_with('engine', 'repo')
my_meta.tables = {'a': 1, 'b': 2}
mock_meta.return_value = my_meta
self.assertRaises( @mock.patch.object(migrate_api, 'db_version')
db_exception.DBMigrationError, migration._migrate_db_version, def test__is_database_under_migrate_control__false(self, mock_db_version):
self.engine, self.path, self.init_version) mock_db_version.side_effect = \
migrate_exceptions.DatabaseNotControlledError()
ret = migration._is_database_under_migrate_control('engine', 'repo')
self.assertFalse(ret)
mock_db_version.assert_called_once_with('engine', 'repo')
@mock.patch.object(migrate_api, 'version_control') @mock.patch.object(alembic_migration.MigrationContext, 'configure')
def test_db_version_raise_not_controlled_error_no_tables(self, mock_vc): def test__is_database_under_alembic_control__true(self, mock_configure):
with mock.patch.object(sqlalchemy, 'MetaData') as mock_meta: context = mock_configure.return_value
self.mock_api_db_version.side_effect = ( context.get_current_revision.return_value = 'foo'
migrate_exception.DatabaseNotControlledError('oups'), engine = mock.MagicMock()
self.init_version) ret = migration._is_database_under_alembic_control(engine)
my_meta = mock.MagicMock() self.assertTrue(ret)
my_meta.tables = {} context.get_current_revision.assert_called_once_with()
mock_meta.return_value = my_meta
migration._migrate_db_version( @mock.patch.object(alembic_migration.MigrationContext, 'configure')
self.engine, self.path, self.init_version) def test__is_database_under_alembic_control__false(self, mock_configure):
context = mock_configure.return_value
mock_vc.assert_called_once_with( context.get_current_revision.return_value = None
self.engine, self.return_value1, self.init_version) engine = mock.MagicMock()
ret = migration._is_database_under_alembic_control(engine)
@mock.patch.object(migrate_api, 'version_control') self.assertFalse(ret)
def test_db_version_raise_not_controlled_alembic_tables(self, mock_vc): context.get_current_revision.assert_called_once_with()
# When there are tables but the alembic control table
# (alembic_version) is present, attempt to version the db.
# This simulates the case where there is are multiple repos (different
# abs_paths) and a different path has been versioned already.
with mock.patch.object(sqlalchemy, 'MetaData') as mock_meta:
self.mock_api_db_version.side_effect = [
migrate_exception.DatabaseNotControlledError('oups'), None]
my_meta = mock.MagicMock()
my_meta.tables = {'alembic_version': 1, 'b': 2}
mock_meta.return_value = my_meta
migration._migrate_db_version(
self.engine, self.path, self.init_version)
mock_vc.assert_called_once_with(
self.engine, self.return_value1, self.init_version)
@mock.patch.object(migrate_api, 'version_control')
def test_db_version_raise_not_controlled_migrate_tables(self, mock_vc):
# When there are tables but the sqlalchemy-migrate control table
# (migrate_version) is present, attempt to version the db.
# This simulates the case where there is are multiple repos (different
# abs_paths) and a different path has been versioned already.
with mock.patch.object(sqlalchemy, 'MetaData') as mock_meta:
self.mock_api_db_version.side_effect = [
migrate_exception.DatabaseNotControlledError('oups'), None]
my_meta = mock.MagicMock()
my_meta.tables = {'migrate_version': 1, 'b': 2}
mock_meta.return_value = my_meta
migration._migrate_db_version(
self.engine, self.path, self.init_version)
mock_vc.assert_called_once_with(
self.engine, self.return_value1, self.init_version)
def test_db_sync_wrong_version(self):
self.assertRaises(
db_exception.DBMigrationError,
migration._migrate_db_sync, self.engine, self.path, 'foo')
@mock.patch.object(migrate_api, 'upgrade')
def test_db_sync_script_not_present(self, upgrade):
# For non existent migration script file sqlalchemy-migrate will raise
# VersionNotFoundError which will be wrapped in DBMigrationError.
upgrade.side_effect = migrate_exception.VersionNotFoundError
self.assertRaises(
db_exception.DBMigrationError,
migration._migrate_db_sync, self.engine, self.path,
self.test_version + 1)
@mock.patch.object(migrate_api, 'upgrade')
def test_db_sync_known_error_raised(self, upgrade):
upgrade.side_effect = migrate_exception.KnownError
self.assertRaises(
db_exception.DBMigrationError,
migration._migrate_db_sync, self.engine, self.path,
self.test_version + 1)
def test_db_sync_upgrade(self):
init_ver = 55
with utils.nested_contexts(
mock.patch.object(migration, '_find_migrate_repo'),
mock.patch.object(migrate_api, 'upgrade')
) as (mock_find_repo, mock_upgrade):
mock_find_repo.return_value = self.return_value
self.mock_api_db_version.return_value = self.test_version - 1
migration._migrate_db_sync(
self.engine, self.path, self.test_version, init_ver)
mock_upgrade.assert_called_once_with(
self.engine, self.return_value, self.test_version)
def test_db_sync_downgrade(self):
with utils.nested_contexts(
mock.patch.object(migration, '_find_migrate_repo'),
mock.patch.object(migrate_api, 'downgrade')
) as (mock_find_repo, mock_downgrade):
mock_find_repo.return_value = self.return_value
self.mock_api_db_version.return_value = self.test_version + 1
migration._migrate_db_sync(
self.engine, self.path, self.test_version)
mock_downgrade.assert_called_once_with(
self.engine, self.return_value, self.test_version)

147
cinder/tests/unit/db/test_migrations.py
View File

@ -11,18 +11,17 @@
# under the License. # under the License.
""" """
Tests for database migrations. This test case reads the configuration Tests for database migrations. For each database backend supported by cinder,
file test_migrations.conf for database connection settings
to use in the tests. For each connection found in the config file,
the test case runs a series of test cases to ensure that migrations work the test case runs a series of test cases to ensure that migrations work
properly both upgrading and downgrading, and that no data loss occurs properly and that no data loss occurs if possible.
if possible.
""" """
import os import os
from alembic import command as alembic_api
from alembic import script as alembic_script
import fixtures import fixtures
from migrate.versioning import api as migration_api from migrate.versioning import api as migrate_api
from migrate.versioning import repository from migrate.versioning import repository
from oslo_db.sqlalchemy import enginefacade from oslo_db.sqlalchemy import enginefacade
from oslo_db.sqlalchemy import test_fixtures from oslo_db.sqlalchemy import test_fixtures
@ -38,7 +37,83 @@ from cinder.tests.unit import utils as test_utils
from cinder.volume import volume_types from cinder.volume import volume_types
class MigrationsMixin(test_migrations.WalkVersionsMixin): class MigrationsWalk(
test_fixtures.OpportunisticDBTestMixin, test_base.BaseTestCase,
):
def setUp(self):
super().setUp()
self.engine = enginefacade.writer.get_engine()
self.config = migration._find_alembic_conf()
self.init_version = migration.ALEMBIC_INIT_VERSION
def _migrate_up(self, revision):
if revision == self.init_version: # no tests for the initial revision
return
self.assertIsNotNone(
getattr(self, '_check_%s' % revision, None),
(
'API DB Migration %s does not have a test; you must add one'
) % revision,
)
alembic_api.upgrade(self.config, revision)
def test_single_base_revision(self):
"""Ensure we only have a single base revision.
There's no good reason for us to have diverging history, so validate
that only one base revision exists. This will prevent simple errors
where people forget to specify the base revision. If this fail for your
change, look for migrations that do not have a 'revises' line in them.
"""
script = alembic_script.ScriptDirectory.from_config(self.config)
self.assertEqual(1, len(script.get_bases()))
def test_single_head_revision(self):
"""Ensure we only have a single head revision.
There's no good reason for us to have diverging history, so validate
that only one head revision exists. This will prevent merge conflicts
adding additional head revision points. If this fail for your change,
look for migrations with the same 'revises' line in them.
"""
script = alembic_script.ScriptDirectory.from_config(self.config)
self.assertEqual(1, len(script.get_heads()))
def test_walk_versions(self):
with self.engine.begin() as connection:
self.config.attributes['connection'] = connection
script = alembic_script.ScriptDirectory.from_config(self.config)
for revision_script in script.walk_revisions():
revision = revision_script.revision
self._migrate_up(revision)
class TestMigrationsWalkSQLite(
MigrationsWalk,
test_fixtures.OpportunisticDBTestMixin,
test_base.BaseTestCase,
):
pass
class TestMigrationsWalkMySQL(
MigrationsWalk,
test_fixtures.OpportunisticDBTestMixin,
test_base.BaseTestCase,
):
FIXTURE = test_fixtures.MySQLOpportunisticFixture
class TestMigrationsWalkPostgreSQL(
MigrationsWalk,
test_fixtures.OpportunisticDBTestMixin,
test_base.BaseTestCase,
):
FIXTURE = test_fixtures.PostgresqlOpportunisticFixture
class LegacyMigrationsWalk(test_migrations.WalkVersionsMixin):
"""Test sqlalchemy-migrate migrations.""" """Test sqlalchemy-migrate migrations."""
BOOL_TYPE = sqlalchemy.types.BOOLEAN BOOL_TYPE = sqlalchemy.types.BOOLEAN
@ -47,9 +122,13 @@ class MigrationsMixin(test_migrations.WalkVersionsMixin):
VARCHAR_TYPE = sqlalchemy.types.VARCHAR VARCHAR_TYPE = sqlalchemy.types.VARCHAR
TEXT_TYPE = sqlalchemy.types.Text TEXT_TYPE = sqlalchemy.types.Text
def setUp(self):
super().setUp()
self.engine = enginefacade.writer.get_engine()
@property @property
def INIT_VERSION(self): def INIT_VERSION(self):
return migration.INIT_VERSION return migration.MIGRATE_INIT_VERSION
@property @property
def REPOSITORY(self): def REPOSITORY(self):
@ -59,16 +138,7 @@ class MigrationsMixin(test_migrations.WalkVersionsMixin):
@property @property
def migration_api(self): def migration_api(self):
return migration_api return migrate_api
def setUp(self):
super(MigrationsMixin, self).setUp()
# (zzzeek) This mixin states that it uses the
# "self.engine" attribute in the migrate_engine() method.
# So the mixin must set that up for itself, oslo_db no longer
# makes these assumptions for you.
self.engine = enginefacade.writer.get_engine()
@property @property
def migrate_engine(self): def migrate_engine(self):
@ -81,7 +151,7 @@ class MigrationsMixin(test_migrations.WalkVersionsMixin):
class BannedDBSchemaOperations(fixtures.Fixture): class BannedDBSchemaOperations(fixtures.Fixture):
"""Ban some operations for migrations""" """Ban some operations for migrations"""
def __init__(self, banned_resources=None): def __init__(self, banned_resources=None):
super(MigrationsMixin.BannedDBSchemaOperations, self).__init__() super().__init__()
self._banned_resources = banned_resources or [] self._banned_resources = banned_resources or []
@staticmethod @staticmethod
@ -92,7 +162,7 @@ class MigrationsMixin(test_migrations.WalkVersionsMixin):
resource, op)) resource, op))
def setUp(self): def setUp(self):
super(MigrationsMixin.BannedDBSchemaOperations, self).setUp() super().setUp()
for thing in self._banned_resources: for thing in self._banned_resources:
self.useFixture(fixtures.MonkeyPatch( self.useFixture(fixtures.MonkeyPatch(
'sqlalchemy.%s.drop' % thing, 'sqlalchemy.%s.drop' % thing,
@ -123,8 +193,9 @@ class MigrationsMixin(test_migrations.WalkVersionsMixin):
banned = ['Table', 'Column'] banned = ['Table', 'Column']
else: else:
banned = None banned = None
with MigrationsMixin.BannedDBSchemaOperations(banned):
super(MigrationsMixin, self).migrate_up(version, with_data) with LegacyMigrationsWalk.BannedDBSchemaOperations(banned):
super().migrate_up(version, with_data)
def __check_cinderbase_fields(self, columns): def __check_cinderbase_fields(self, columns):
"""Check fields inherited from CinderBase ORM class.""" """Check fields inherited from CinderBase ORM class."""
@ -217,9 +288,11 @@ class MigrationsMixin(test_migrations.WalkVersionsMixin):
self.assert_each_foreign_key_is_part_of_an_index() self.assert_each_foreign_key_is_part_of_an_index()
class TestSqliteMigrations(test_fixtures.OpportunisticDBTestMixin, class TestLegacyMigrationsWalkSQLite(
MigrationsMixin, test_fixtures.OpportunisticDBTestMixin,
test_base.BaseTestCase): LegacyMigrationsWalk,
test_base.BaseTestCase,
):
def assert_each_foreign_key_is_part_of_an_index(self): def assert_each_foreign_key_is_part_of_an_index(self):
# Skip the test for SQLite because SQLite does not list # Skip the test for SQLite because SQLite does not list
@ -228,9 +301,11 @@ class TestSqliteMigrations(test_fixtures.OpportunisticDBTestMixin,
pass pass
class TestMysqlMigrations(test_fixtures.OpportunisticDBTestMixin, class TestLegacyMigrationsWalkMySQL(
MigrationsMixin, test_fixtures.OpportunisticDBTestMixin,
test_base.BaseTestCase): LegacyMigrationsWalk,
test_base.BaseTestCase,
):
FIXTURE = test_fixtures.MySQLOpportunisticFixture FIXTURE = test_fixtures.MySQLOpportunisticFixture
BOOL_TYPE = sqlalchemy.dialects.mysql.TINYINT BOOL_TYPE = sqlalchemy.dialects.mysql.TINYINT
@ -241,7 +316,10 @@ class TestMysqlMigrations(test_fixtures.OpportunisticDBTestMixin,
# add this to the global lists to make reset work with it, it's removed # add this to the global lists to make reset work with it, it's removed
# automatically in tearDown so no need to clean it up here. # automatically in tearDown so no need to clean it up here.
# sanity check # sanity check
migration.db_sync(engine=self.migrate_engine) repo = migration._find_migrate_repo()
migrate_api.version_control(
self.migrate_engine, repo, migration.MIGRATE_INIT_VERSION)
migrate_api.upgrade(self.migrate_engine, repo)
total = self.migrate_engine.execute( total = self.migrate_engine.execute(
"SELECT count(*) " "SELECT count(*) "
@ -268,13 +346,14 @@ class TestMysqlMigrations(test_fixtures.OpportunisticDBTestMixin,
# Depending on the MariaDB version, and the page size, we may not have # Depending on the MariaDB version, and the page size, we may not have
# been able to change quota_usage_resource to 300 chars, it could still # been able to change quota_usage_resource to 300 chars, it could still
# be 255. # be 255.
self.assertIn(quota_usage_resource.c.resource.type.length, self.assertIn(quota_usage_resource.c.resource.type.length, (255, 300))
(255, 300))
class TestPostgresqlMigrations(test_fixtures.OpportunisticDBTestMixin, class TestLegacyMigrationsWalkPostgreSQL(
MigrationsMixin, test_fixtures.OpportunisticDBTestMixin,
test_base.BaseTestCase): LegacyMigrationsWalk,
test_base.BaseTestCase,
):
FIXTURE = test_fixtures.PostgresqlOpportunisticFixture FIXTURE = test_fixtures.PostgresqlOpportunisticFixture
TIME_TYPE = sqlalchemy.types.TIMESTAMP TIME_TYPE = sqlalchemy.types.TIMESTAMP

25
releasenotes/notes/switch-to-alembic-2bbe27749fde70ff.yaml Normal file
View File

@ -0,0 +1,25 @@
---
upgrade:
- |
The database migration engine has changed from `sqlalchemy-migrate`__ to
`alembic`__. For most deployments, this should have minimal to no impact
and the switch should be mostly transparent. The main user-facing impact is
the change in schema versioning. While sqlalchemy-migrate used a linear,
integer-based versioning scheme, which required placeholder migrations to
allow for potential migration backports, alembic uses a distributed version
control-like schema where a migration's ancestor is encoded in the file and
branches are possible. The alembic migration files therefore use a
arbitrary UUID-like naming scheme and the ``cinder-manage db sync`` command
now expects such an version when manually specifying the version that should
be applied. For example::
$ cinder-manage db sync 921e1a36b076
It is no longer possible to specify an sqlalchemy-migrate-based version.
When the ``cinder-manage db sync`` command is run, all remaining
sqlalchemy-migrate-based migrations will be automatically applied.
Attempting to specify an sqlalchemy-migrate-based version will result in an
error.
.. __: https://sqlalchemy-migrate.readthedocs.io/en/latest/
.. __: https://alembic.sqlalchemy.org/en/latest/