Transition from migrate to alembic
Toolkit for smooth transitioning from migrate tool to alembic. Check README for some description POC: Integration with nova-manage https://review.openstack.org/#/c/60517/ Blueprint: oslo-migration-toolkit Change-Id: Icfa7d8736a84da1a4890eb417d48bab4c73e1f89
This commit is contained in:
Dima Shulyak
parent
0b117700f8
commit
6eb39221e2
|
|
@ -230,7 +230,7 @@ def db_version(abs_path, init_version):
|
|||
engine = get_engine()
|
||||
meta.reflect(bind=engine)
|
||||
tables = meta.tables
|
||||
if len(tables) == 0:
|
||||
if len(tables) == 0 or 'alembic_version' in tables:
|
||||
db_version_control(abs_path, init_version)
|
||||
return versioning_api.db_version(get_engine(), repository)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
This module could be used either for:
|
||||
1. Smooth transition from migrate tool to alembic
|
||||
2. As standalone alembic tool
|
||||
|
||||
Core points:
|
||||
1. Upgrade/downgrade database with usage of alembic/migrate migrations
|
||||
or both
|
||||
2. Compatibility with oslo.config
|
||||
3. The way to autogenerate new revisions or stamps
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
|
||||
import alembic
|
||||
from alembic import config as alembic_config
|
||||
import alembic.migration as alembic_migration
|
||||
|
||||
from openstack.common.db.sqlalchemy.migration_cli import ext_base
|
||||
from openstack.common.db.sqlalchemy import session as db_session
|
||||
|
||||
|
||||
class AlembicExtension(ext_base.MigrationExtensionBase):
|
||||
|
||||
order = 2
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
return os.path.exists(self.alembic_ini_path)
|
||||
|
||||
def __init__(self, migration_config):
|
||||
"""Extension to provide alembic features.
|
||||
|
||||
:param migration_config: Stores specific configuration for migrations
|
||||
:type migration_config: dict
|
||||
"""
|
||||
self.alembic_ini_path = migration_config.get('alembic_ini_path', '')
|
||||
self.config = alembic_config.Config(self.alembic_ini_path)
|
||||
# option should be used if script is not in default directory
|
||||
repo_path = migration_config.get('alembic_repo_path')
|
||||
if repo_path:
|
||||
self.config.set_main_option('script_location', repo_path)
|
||||
|
||||
def upgrade(self, version):
|
||||
return alembic.command.upgrade(self.config, version or 'head')
|
||||
|
||||
def downgrade(self, version):
|
||||
if isinstance(version, int) or version is None or version.isdigit():
|
||||
version = 'base'
|
||||
return alembic.command.downgrade(self.config, version)
|
||||
|
||||
def version(self):
|
||||
engine = db_session.get_engine()
|
||||
with engine.connect() as conn:
|
||||
context = alembic_migration.MigrationContext.configure(conn)
|
||||
return context.get_current_revision()
|
||||
|
||||
def revision(self, message='', autogenerate=False):
|
||||
"""Creates template for migration.
|
||||
|
||||
:param message: Text that will be used for migration title
|
||||
:type message: string
|
||||
:param autogenerate: If True - generates diff based on current database
|
||||
state
|
||||
:type autogenerate: bool
|
||||
"""
|
||||
return alembic.command.revision(self.config, message=message,
|
||||
autogenerate=autogenerate)
|
||||
|
||||
def stamp(self, revision):
|
||||
"""Stamps database with provided revision.
|
||||
|
||||
:param revision: Should match one from repository or head - to stamp
|
||||
database with most recent revision
|
||||
:type revision: string
|
||||
"""
|
||||
return alembic.command.stamp(self.config, revision=revision)
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import abc
|
||||
|
||||
import six
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
class MigrationExtensionBase(object):
|
||||
|
||||
#used to sort migration in logical order
|
||||
order = 0
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
"""Used for availability verification of a plugin.
|
||||
|
||||
:rtype: bool
|
||||
"""
|
||||
return False
|
||||
|
||||
@abc.abstractmethod
|
||||
def upgrade(self, version):
|
||||
"""Used for upgrading database.
|
||||
|
||||
:param version: Desired database version
|
||||
:type version: string
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def downgrade(self, version):
|
||||
"""Used for downgrading database.
|
||||
|
||||
:param version: Desired database version
|
||||
:type version: string
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def version(self):
|
||||
"""Current database version.
|
||||
|
||||
:returns: Databse version
|
||||
:rtype: string
|
||||
"""
|
||||
|
||||
def revision(self, *args, **kwargs):
|
||||
"""Used to generate migration script.
|
||||
|
||||
In migration engines that support this feature, it should generate
|
||||
new migration script.
|
||||
|
||||
Accept arbitrary set of arguments.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def stamp(self, *args, **kwargs):
|
||||
"""Stamps database based on plugin features.
|
||||
|
||||
Accept arbitrary set of arguments.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def __cmp__(self, other):
|
||||
"""Used for definition of plugin order.
|
||||
|
||||
:param other: MigrationExtensionBase instance
|
||||
:rtype: bool
|
||||
"""
|
||||
return self.order > other.order
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
|
||||
from openstack.common.db.sqlalchemy import migration
|
||||
from openstack.common.db.sqlalchemy.migration_cli import ext_base
|
||||
from openstack.common.gettextutils import _ # noqa
|
||||
from openstack.common import log as logging
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MigrateExtension(ext_base.MigrationExtensionBase):
|
||||
"""Extension to provide sqlalchemy-migrate features.
|
||||
|
||||
:param migration_config: Stores specific configuration for migrations
|
||||
:type migration_config: dict
|
||||
"""
|
||||
|
||||
order = 1
|
||||
|
||||
def __init__(self, migration_config):
|
||||
self.repository = migration_config.get('migration_repo_path', '')
|
||||
self.init_version = migration_config.get('init_version', 0)
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
return os.path.exists(self.repository)
|
||||
|
||||
def upgrade(self, version):
|
||||
version = None if version == 'head' else version
|
||||
return migration.db_sync(
|
||||
self.repository, version,
|
||||
init_version=self.init_version)
|
||||
|
||||
def downgrade(self, version):
|
||||
try:
|
||||
#version for migrate should be valid int - else skip
|
||||
if version in ('base', None):
|
||||
version = self.init_version
|
||||
version = int(version)
|
||||
return migration.db_sync(
|
||||
self.repository, version,
|
||||
init_version=self.init_version)
|
||||
except ValueError:
|
||||
LOG.error(
|
||||
_('Migration number for migrate plugin must be valid '
|
||||
'integer or empty, if you want to downgrade '
|
||||
'to initial state')
|
||||
)
|
||||
raise
|
||||
|
||||
def version(self):
|
||||
return migration.db_version(
|
||||
self.repository, init_version=self.init_version)
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from stevedore import enabled
|
||||
|
||||
|
||||
MIGRATION_NAMESPACE = 'openstack.common.migration'
|
||||
|
||||
|
||||
def check_plugin_enabled(ext):
|
||||
"""Used for EnabledExtensionManager"""
|
||||
return ext.obj.enabled
|
||||
|
||||
|
||||
class MigrationManager(object):
|
||||
|
||||
def __init__(self, migration_config):
|
||||
self._manager = enabled.EnabledExtensionManager(
|
||||
MIGRATION_NAMESPACE,
|
||||
check_plugin_enabled,
|
||||
invoke_kwds={'migration_config': migration_config},
|
||||
invoke_on_load=True
|
||||
)
|
||||
if not self._plugins:
|
||||
raise ValueError('There must be at least one plugin active.')
|
||||
|
||||
@property
|
||||
def _plugins(self):
|
||||
return sorted(ext.obj for ext in self._manager.extensions)
|
||||
|
||||
def upgrade(self, revision):
|
||||
"""Upgrade database with all available backends."""
|
||||
results = []
|
||||
for plugin in self._plugins:
|
||||
results.append(plugin.upgrade(revision))
|
||||
return results
|
||||
|
||||
def downgrade(self, revision):
|
||||
"""Downgrade database with available backends."""
|
||||
#downgrading should be performed in reversed order
|
||||
results = []
|
||||
for plugin in reversed(self._plugins):
|
||||
results.append(plugin.downgrade(revision))
|
||||
return results
|
||||
|
||||
def version(self):
|
||||
"""Return last version of db."""
|
||||
last = None
|
||||
for plugin in self._plugins:
|
||||
version = plugin.version()
|
||||
if version:
|
||||
last = version
|
||||
return last
|
||||
|
||||
def revision(self, message, autogenerate):
|
||||
"""Generate template or autogenerated revision."""
|
||||
#revision should be done only by last plugin
|
||||
return self._plugins[-1].revision(message, autogenerate)
|
||||
|
||||
def stamp(self, revision):
|
||||
"""Create stamp for a given revision."""
|
||||
return self._plugins[-1].stamp(revision)
|
||||
|
|
@ -0,0 +1,212 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import mock
|
||||
|
||||
from openstack.common.db.sqlalchemy.migration_cli import ext_alembic
|
||||
from openstack.common.db.sqlalchemy.migration_cli import ext_migrate
|
||||
from openstack.common.db.sqlalchemy.migration_cli import manager
|
||||
from openstack.common import test
|
||||
|
||||
|
||||
class MockWithCmp(mock.MagicMock):
|
||||
|
||||
order = 0
|
||||
|
||||
def __cmp__(self, other):
|
||||
return self.order > other.order
|
||||
|
||||
|
||||
@mock.patch(('openstack.common.db.sqlalchemy.migration_cli.'
|
||||
'ext_alembic.alembic.command'))
|
||||
class TestAlembicExtension(test.BaseTestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.migration_config = {'alembic_ini_path': '.'}
|
||||
self.alembic = ext_alembic.AlembicExtension(self.migration_config)
|
||||
super(TestAlembicExtension, self).setUp()
|
||||
|
||||
def test_check_enabled_true(self, command):
|
||||
"""Verifies that enabled returns True on non empty
|
||||
alembic_ini_path conf variable
|
||||
"""
|
||||
self.assertTrue(self.alembic.enabled)
|
||||
|
||||
def test_check_enabled_false(self, command):
|
||||
"""Verifies enabled returns False on empty
|
||||
alembic_ini_path variable
|
||||
"""
|
||||
self.migration_config['alembic_ini_path'] = ''
|
||||
alembic = ext_alembic.AlembicExtension(self.migration_config)
|
||||
self.assertFalse(alembic.enabled)
|
||||
|
||||
def test_upgrade_none(self, command):
|
||||
self.alembic.upgrade(None)
|
||||
command.upgrade.assert_called_once_with(self.alembic.config, 'head')
|
||||
|
||||
def test_upgrade_normal(self, command):
|
||||
self.alembic.upgrade('131daa')
|
||||
command.upgrade.assert_called_once_with(self.alembic.config, '131daa')
|
||||
|
||||
def test_downgrade_none(self, command):
|
||||
self.alembic.downgrade(None)
|
||||
command.downgrade.assert_called_once_with(self.alembic.config, 'base')
|
||||
|
||||
def test_downgrade_int(self, command):
|
||||
self.alembic.downgrade(111)
|
||||
command.downgrade.assert_called_once_with(self.alembic.config, 'base')
|
||||
|
||||
def test_downgrade_normal(self, command):
|
||||
self.alembic.downgrade('131daa')
|
||||
command.downgrade.assert_called_once_with(
|
||||
self.alembic.config, '131daa')
|
||||
|
||||
def test_revision(self, command):
|
||||
self.alembic.revision(message='test', autogenerate=True)
|
||||
command.revision.assert_called_once_with(
|
||||
self.alembic.config, message='test', autogenerate=True)
|
||||
|
||||
def test_stamp(self, command):
|
||||
self.alembic.stamp('stamp')
|
||||
command.stamp.assert_called_once_with(
|
||||
self.alembic.config, revision='stamp')
|
||||
|
||||
def test_version(self, command):
|
||||
version = self.alembic.version()
|
||||
self.assertIsNone(version)
|
||||
|
||||
|
||||
@mock.patch(('openstack.common.db.sqlalchemy.migration_cli.'
|
||||
'ext_migrate.migration'))
|
||||
class TestMigrateExtension(test.BaseTestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.migration_config = {'migration_repo_path': '.'}
|
||||
self.migrate = ext_migrate.MigrateExtension(self.migration_config)
|
||||
super(TestMigrateExtension, self).setUp()
|
||||
|
||||
def test_check_enabled_true(self, migration):
|
||||
self.assertTrue(self.migrate.enabled)
|
||||
|
||||
def test_check_enabled_false(self, migration):
|
||||
self.migration_config['migration_repo_path'] = ''
|
||||
migrate = ext_migrate.MigrateExtension(self.migration_config)
|
||||
self.assertFalse(migrate.enabled)
|
||||
|
||||
def test_upgrade_head(self, migration):
|
||||
self.migrate.upgrade('head')
|
||||
migration.db_sync.assert_called_once_with(
|
||||
self.migrate.repository, None, init_version=0)
|
||||
|
||||
def test_upgrade_normal(self, migration):
|
||||
self.migrate.upgrade(111)
|
||||
migration.db_sync.assert_called_once_with(
|
||||
self.migrate.repository, 111, init_version=0)
|
||||
|
||||
def test_downgrade_init_version_from_base(self, migration):
|
||||
self.migrate.downgrade('base')
|
||||
migration.db_sync.assert_called_once_with(
|
||||
self.migrate.repository, mock.ANY,
|
||||
init_version=mock.ANY)
|
||||
|
||||
def test_downgrade_init_version_from_none(self, migration):
|
||||
self.migrate.downgrade(None)
|
||||
migration.db_sync.assert_called_once_with(
|
||||
self.migrate.repository, mock.ANY,
|
||||
init_version=mock.ANY)
|
||||
|
||||
def test_downgrade_normal(self, migration):
|
||||
self.migrate.downgrade(101)
|
||||
migration.db_sync.assert_called_once_with(
|
||||
self.migrate.repository, 101, init_version=0)
|
||||
|
||||
def test_version(self, migration):
|
||||
self.migrate.version()
|
||||
migration.db_version.assert_called_once_with(
|
||||
self.migrate.repository, init_version=0)
|
||||
|
||||
def test_change_init_version(self, migration):
|
||||
self.migration_config['init_version'] = 101
|
||||
migrate = ext_migrate.MigrateExtension(self.migration_config)
|
||||
migrate.downgrade(None)
|
||||
migration.db_sync.assert_called_once_with(
|
||||
self.migrate.repository,
|
||||
self.migration_config['init_version'],
|
||||
init_version=self.migration_config['init_version'])
|
||||
|
||||
|
||||
class TestMigrationManager(test.BaseTestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.migration_config = {'alembic_ini_path': '.',
|
||||
'migrate_repo_path': '.'}
|
||||
self.migration_manager = manager.MigrationManager(
|
||||
self.migration_config)
|
||||
self.ext = mock.Mock()
|
||||
self.migration_manager._manager.extensions = [self.ext]
|
||||
super(TestMigrationManager, self).setUp()
|
||||
|
||||
def test_manager_update(self):
|
||||
self.migration_manager.upgrade('head')
|
||||
self.ext.obj.upgrade.assert_called_once_with('head')
|
||||
|
||||
def test_manager_update_revision_none(self):
|
||||
self.migration_manager.upgrade(None)
|
||||
self.ext.obj.upgrade.assert_called_once_with(None)
|
||||
|
||||
def test_downgrade_normal_revision(self):
|
||||
self.migration_manager.downgrade('111abcd')
|
||||
self.ext.obj.downgrade.assert_called_once_with('111abcd')
|
||||
|
||||
def test_version(self):
|
||||
self.migration_manager.version()
|
||||
self.ext.obj.version.assert_called_once_with()
|
||||
|
||||
def test_revision_message_autogenerate(self):
|
||||
self.migration_manager.revision('test', True)
|
||||
self.ext.obj.revision.assert_called_once_with('test', True)
|
||||
|
||||
def test_revision_only_message(self):
|
||||
self.migration_manager.revision('test', False)
|
||||
self.ext.obj.revision.assert_called_once_with('test', False)
|
||||
|
||||
def test_stamp(self):
|
||||
self.migration_manager.stamp('stamp')
|
||||
self.ext.obj.stamp.assert_called_once_with('stamp')
|
||||
|
||||
|
||||
class TestMigrationRightOrder(test.BaseTestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.migration_config = {'alembic_ini_path': '.',
|
||||
'migrate_repo_path': '.'}
|
||||
self.migration_manager = manager.MigrationManager(
|
||||
self.migration_config)
|
||||
self.first_ext = MockWithCmp()
|
||||
self.first_ext.obj.order = 1
|
||||
self.first_ext.obj.upgrade.return_value = 100
|
||||
self.first_ext.obj.downgrade.return_value = 0
|
||||
self.second_ext = MockWithCmp()
|
||||
self.second_ext.obj.order = 2
|
||||
self.second_ext.obj.upgrade.return_value = 200
|
||||
self.second_ext.obj.downgrade.return_value = 100
|
||||
self.migration_manager._manager.extensions = [self.first_ext,
|
||||
self.second_ext]
|
||||
super(TestMigrationRightOrder, self).setUp()
|
||||
|
||||
def test_upgrade_right_order(self):
|
||||
results = self.migration_manager.upgrade(None)
|
||||
self.assertEqual(results, [100, 200])
|
||||
|
||||
def test_downgrade_right_order(self):
|
||||
results = self.migration_manager.downgrade(None)
|
||||
self.assertEqual(results, [100, 0])
|
||||
Loading…
Reference in New Issue