sql: Prepare for alembic migration
Nothing functional here. We simply switch from using "repo_name" terminology and move/rename some helper functions. Branches aren't really a SQLAlchemy-Migrate thing but it's close enough to do. Change-Id: I005d20ef21b6c8122be90e8afb38abd902fdfc6e Signed-off-by: Stephen Finucane <stephenfin@redhat.com>
This commit is contained in:
Stephen Finucane
parent
a6887c13ea
commit
dce38678fc
|
|
@ -277,7 +277,7 @@ class DbSync(BaseApp):
|
|||
def check_db_sync_status(cls):
|
||||
status = 0
|
||||
try:
|
||||
expand_version = upgrades.get_db_version(repo='expand_repo')
|
||||
expand_version = upgrades.get_db_version(branch='expand')
|
||||
except db_exception.DBMigrationError:
|
||||
LOG.info(
|
||||
'Your database is not currently under version '
|
||||
|
|
@ -288,12 +288,12 @@ class DbSync(BaseApp):
|
|||
|
||||
try:
|
||||
migrate_version = upgrades.get_db_version(
|
||||
repo='data_migration_repo')
|
||||
branch='data_migration')
|
||||
except db_exception.DBMigrationError:
|
||||
migrate_version = 0
|
||||
|
||||
try:
|
||||
contract_version = upgrades.get_db_version(repo='contract_repo')
|
||||
contract_version = upgrades.get_db_version(branch='contract')
|
||||
except db_exception.DBMigrationError:
|
||||
contract_version = 0
|
||||
|
||||
|
|
|
|||
|
|
@ -28,17 +28,17 @@ from keystone.i18n import _
|
|||
|
||||
INITIAL_VERSION = 72
|
||||
LATEST_VERSION = 79
|
||||
EXPAND_REPO = 'expand_repo'
|
||||
DATA_MIGRATION_REPO = 'data_migration_repo'
|
||||
CONTRACT_REPO = 'contract_repo'
|
||||
EXPAND_BRANCH = 'expand'
|
||||
DATA_MIGRATION_BRANCH = 'data_migration'
|
||||
CONTRACT_BRANCH = 'contract'
|
||||
|
||||
|
||||
def _get_migrate_repo_path(repo_name):
|
||||
def _get_migrate_repo_path(branch):
|
||||
abs_path = os.path.abspath(
|
||||
os.path.join(
|
||||
os.path.dirname(sql.__file__),
|
||||
'legacy_migrations',
|
||||
repo_name,
|
||||
f'{branch}_repo',
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -142,8 +142,18 @@ def _migrate_db_sync(engine, abs_path, version=None, init_version=0):
|
|||
return migrate_api.downgrade(engine, repository, version)
|
||||
|
||||
|
||||
def _sync_repo(repo_name):
|
||||
abs_path = _get_migrate_repo_path(repo_name)
|
||||
def get_db_version(branch=EXPAND_BRANCH):
|
||||
abs_path = _get_migrate_repo_path(branch)
|
||||
with sql.session_for_read() as session:
|
||||
return _migrate_db_version(
|
||||
session.get_bind(),
|
||||
abs_path,
|
||||
INITIAL_VERSION,
|
||||
)
|
||||
|
||||
|
||||
def _db_sync(branch):
|
||||
abs_path = _get_migrate_repo_path(branch)
|
||||
with sql.session_for_write() as session:
|
||||
engine = session.get_bind()
|
||||
_migrate_db_sync(
|
||||
|
|
@ -153,6 +163,80 @@ def _sync_repo(repo_name):
|
|||
)
|
||||
|
||||
|
||||
def _validate_upgrade_order(branch, target_repo_version=None):
|
||||
"""Validate the state of the migration repositories.
|
||||
|
||||
This is run before allowing the db_sync command to execute. Ensure the
|
||||
upgrade step and version specified by the operator remains consistent with
|
||||
the upgrade process. I.e. expand's version is greater or equal to
|
||||
migrate's, migrate's version is greater or equal to contract's.
|
||||
|
||||
:param branch: The name of the repository that the user is trying to
|
||||
upgrade.
|
||||
:param target_repo_version: The version to upgrade the repo. Otherwise, the
|
||||
version will be upgraded to the latest version
|
||||
available.
|
||||
"""
|
||||
# Initialize a dict to have each key assigned a repo with their value being
|
||||
# the repo that comes before.
|
||||
db_sync_order = {
|
||||
DATA_MIGRATION_BRANCH: EXPAND_BRANCH,
|
||||
CONTRACT_BRANCH: DATA_MIGRATION_BRANCH,
|
||||
}
|
||||
|
||||
if branch == EXPAND_BRANCH:
|
||||
return
|
||||
|
||||
# find the latest version that the current command will upgrade to if there
|
||||
# wasn't a version specified for upgrade.
|
||||
if not target_repo_version:
|
||||
abs_path = _get_migrate_repo_path(branch)
|
||||
repo = _find_migrate_repo(abs_path)
|
||||
target_repo_version = int(repo.latest)
|
||||
|
||||
# get current version of the command that runs before the current command.
|
||||
dependency_repo_version = get_db_version(branch=db_sync_order[branch])
|
||||
|
||||
if dependency_repo_version < target_repo_version:
|
||||
raise db_exception.DBMigrationError(
|
||||
'You are attempting to upgrade %s ahead of %s. Please refer to '
|
||||
'https://docs.openstack.org/keystone/latest/admin/'
|
||||
'identity-upgrading.html '
|
||||
'to see the proper steps for rolling upgrades.' % (
|
||||
branch, db_sync_order[branch]))
|
||||
|
||||
|
||||
def expand_schema():
|
||||
"""Expand the database schema ahead of data migration.
|
||||
|
||||
This is run manually by the keystone-manage command before the first
|
||||
keystone node is migrated to the latest release.
|
||||
"""
|
||||
_validate_upgrade_order(EXPAND_BRANCH)
|
||||
_db_sync(branch=EXPAND_BRANCH)
|
||||
|
||||
|
||||
def migrate_data():
|
||||
"""Migrate data to match the new schema.
|
||||
|
||||
This is run manually by the keystone-manage command once the keystone
|
||||
schema has been expanded for the new release.
|
||||
"""
|
||||
_validate_upgrade_order(DATA_MIGRATION_BRANCH)
|
||||
_db_sync(branch=DATA_MIGRATION_BRANCH)
|
||||
|
||||
|
||||
def contract_schema():
|
||||
"""Contract the database.
|
||||
|
||||
This is run manually by the keystone-manage command once the keystone
|
||||
nodes have been upgraded to the latest release and will remove any old
|
||||
tables/columns that are no longer required.
|
||||
"""
|
||||
_validate_upgrade_order(CONTRACT_BRANCH)
|
||||
_db_sync(branch=CONTRACT_BRANCH)
|
||||
|
||||
|
||||
def offline_sync_database_to_version(version=None):
|
||||
"""Perform and off-line sync of the database.
|
||||
|
||||
|
|
@ -171,87 +255,3 @@ def offline_sync_database_to_version(version=None):
|
|||
expand_schema()
|
||||
migrate_data()
|
||||
contract_schema()
|
||||
|
||||
|
||||
def get_db_version(repo=EXPAND_REPO):
|
||||
abs_path = _get_migrate_repo_path(repo)
|
||||
with sql.session_for_read() as session:
|
||||
return _migrate_db_version(
|
||||
session.get_bind(),
|
||||
abs_path,
|
||||
INITIAL_VERSION,
|
||||
)
|
||||
|
||||
|
||||
def validate_upgrade_order(repo_name, target_repo_version=None):
|
||||
"""Validate the state of the migration repositories.
|
||||
|
||||
This is run before allowing the db_sync command to execute. Ensure the
|
||||
upgrade step and version specified by the operator remains consistent with
|
||||
the upgrade process. I.e. expand's version is greater or equal to
|
||||
migrate's, migrate's version is greater or equal to contract's.
|
||||
|
||||
:param repo_name: The name of the repository that the user is trying to
|
||||
upgrade.
|
||||
:param target_repo_version: The version to upgrade the repo. Otherwise, the
|
||||
version will be upgraded to the latest version
|
||||
available.
|
||||
"""
|
||||
# Initialize a dict to have each key assigned a repo with their value being
|
||||
# the repo that comes before.
|
||||
db_sync_order = {
|
||||
DATA_MIGRATION_REPO: EXPAND_REPO,
|
||||
CONTRACT_REPO: DATA_MIGRATION_REPO,
|
||||
}
|
||||
|
||||
if repo_name == EXPAND_REPO:
|
||||
return
|
||||
|
||||
# find the latest version that the current command will upgrade to if there
|
||||
# wasn't a version specified for upgrade.
|
||||
if not target_repo_version:
|
||||
abs_path = _get_migrate_repo_path(repo_name)
|
||||
repo = _find_migrate_repo(abs_path)
|
||||
target_repo_version = int(repo.latest)
|
||||
|
||||
# get current version of the command that runs before the current command.
|
||||
dependency_repo_version = get_db_version(repo=db_sync_order[repo_name])
|
||||
|
||||
if dependency_repo_version < target_repo_version:
|
||||
raise db_exception.DBMigrationError(
|
||||
'You are attempting to upgrade %s ahead of %s. Please refer to '
|
||||
'https://docs.openstack.org/keystone/latest/admin/'
|
||||
'identity-upgrading.html '
|
||||
'to see the proper steps for rolling upgrades.' % (
|
||||
repo_name, db_sync_order[repo_name]))
|
||||
|
||||
|
||||
def expand_schema():
|
||||
"""Expand the database schema ahead of data migration.
|
||||
|
||||
This is run manually by the keystone-manage command before the first
|
||||
keystone node is migrated to the latest release.
|
||||
"""
|
||||
validate_upgrade_order(EXPAND_REPO)
|
||||
_sync_repo(repo_name=EXPAND_REPO)
|
||||
|
||||
|
||||
def migrate_data():
|
||||
"""Migrate data to match the new schema.
|
||||
|
||||
This is run manually by the keystone-manage command once the keystone
|
||||
schema has been expanded for the new release.
|
||||
"""
|
||||
validate_upgrade_order(DATA_MIGRATION_REPO)
|
||||
_sync_repo(repo_name=DATA_MIGRATION_REPO)
|
||||
|
||||
|
||||
def contract_schema():
|
||||
"""Contract the database.
|
||||
|
||||
This is run manually by the keystone-manage command once the keystone
|
||||
nodes have been upgraded to the latest release and will remove any old
|
||||
tables/columns that are no longer required.
|
||||
"""
|
||||
validate_upgrade_order(CONTRACT_REPO)
|
||||
_sync_repo(repo_name=CONTRACT_REPO)
|
||||
|
|
|
|||
|
|
@ -248,7 +248,7 @@ class Repository:
|
|||
version = migrate_api._migrate_version(
|
||||
self.schema_, version, upgrade, err,
|
||||
)
|
||||
upgrades.validate_upgrade_order(
|
||||
upgrades._validate_upgrade_order(
|
||||
self.repo_name, target_repo_version=version,
|
||||
)
|
||||
if not current_schema:
|
||||
|
|
@ -305,28 +305,28 @@ class MigrateBase(
|
|||
self.addCleanup(sql.cleanup)
|
||||
|
||||
self.repos = {
|
||||
upgrades.EXPAND_REPO: Repository(
|
||||
self.engine, upgrades.EXPAND_REPO,
|
||||
upgrades.EXPAND_BRANCH: Repository(
|
||||
self.engine, upgrades.EXPAND_BRANCH,
|
||||
),
|
||||
upgrades.DATA_MIGRATION_REPO: Repository(
|
||||
self.engine, upgrades.DATA_MIGRATION_REPO,
|
||||
upgrades.DATA_MIGRATION_BRANCH: Repository(
|
||||
self.engine, upgrades.DATA_MIGRATION_BRANCH,
|
||||
),
|
||||
upgrades.CONTRACT_REPO: Repository(
|
||||
self.engine, upgrades.CONTRACT_REPO,
|
||||
upgrades.CONTRACT_BRANCH: Repository(
|
||||
self.engine, upgrades.CONTRACT_BRANCH,
|
||||
),
|
||||
}
|
||||
|
||||
def expand(self, *args, **kwargs):
|
||||
"""Expand database schema."""
|
||||
self.repos[upgrades.EXPAND_REPO].upgrade(*args, **kwargs)
|
||||
self.repos[upgrades.EXPAND_BRANCH].upgrade(*args, **kwargs)
|
||||
|
||||
def migrate(self, *args, **kwargs):
|
||||
"""Migrate data."""
|
||||
self.repos[upgrades.DATA_MIGRATION_REPO].upgrade(*args, **kwargs)
|
||||
self.repos[upgrades.DATA_MIGRATION_BRANCH].upgrade(*args, **kwargs)
|
||||
|
||||
def contract(self, *args, **kwargs):
|
||||
"""Contract database schema."""
|
||||
self.repos[upgrades.CONTRACT_REPO].upgrade(*args, **kwargs)
|
||||
self.repos[upgrades.CONTRACT_BRANCH].upgrade(*args, **kwargs)
|
||||
|
||||
@property
|
||||
def metadata(self):
|
||||
|
|
@ -362,8 +362,8 @@ class ExpandSchemaUpgradeTests(MigrateBase):
|
|||
|
||||
def test_start_version_db_init_version(self):
|
||||
self.assertEqual(
|
||||
self.repos[upgrades.EXPAND_REPO].min_version,
|
||||
self.repos[upgrades.EXPAND_REPO].version)
|
||||
self.repos[upgrades.EXPAND_BRANCH].min_version,
|
||||
self.repos[upgrades.EXPAND_BRANCH].version)
|
||||
|
||||
def test_blank_db_to_start(self):
|
||||
self.assertTableDoesNotExist('user')
|
||||
|
|
@ -399,8 +399,8 @@ class DataMigrationUpgradeTests(MigrateBase):
|
|||
|
||||
def test_start_version_db_init_version(self):
|
||||
self.assertEqual(
|
||||
self.repos[upgrades.DATA_MIGRATION_REPO].min_version,
|
||||
self.repos[upgrades.DATA_MIGRATION_REPO].version,
|
||||
self.repos[upgrades.DATA_MIGRATION_BRANCH].min_version,
|
||||
self.repos[upgrades.DATA_MIGRATION_BRANCH].version,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -435,8 +435,8 @@ class ContractSchemaUpgradeTests(MigrateBase, unit.TestCase):
|
|||
|
||||
def test_start_version_db_init_version(self):
|
||||
self.assertEqual(
|
||||
self.repos[upgrades.CONTRACT_REPO].min_version,
|
||||
self.repos[upgrades.CONTRACT_REPO].version,
|
||||
self.repos[upgrades.CONTRACT_BRANCH].min_version,
|
||||
self.repos[upgrades.CONTRACT_BRANCH].version,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -476,12 +476,12 @@ class VersionTests(MigrateBase):
|
|||
"""
|
||||
# Transitive comparison: expand == data migration == contract
|
||||
self.assertEqual(
|
||||
self.repos[upgrades.EXPAND_REPO].max_version,
|
||||
self.repos[upgrades.DATA_MIGRATION_REPO].max_version,
|
||||
self.repos[upgrades.EXPAND_BRANCH].max_version,
|
||||
self.repos[upgrades.DATA_MIGRATION_BRANCH].max_version,
|
||||
)
|
||||
self.assertEqual(
|
||||
self.repos[upgrades.DATA_MIGRATION_REPO].max_version,
|
||||
self.repos[upgrades.CONTRACT_REPO].max_version,
|
||||
self.repos[upgrades.DATA_MIGRATION_BRANCH].max_version,
|
||||
self.repos[upgrades.CONTRACT_BRANCH].max_version,
|
||||
)
|
||||
|
||||
def test_migrate_repos_file_names_have_prefix(self):
|
||||
|
|
@ -496,17 +496,17 @@ class VersionTests(MigrateBase):
|
|||
versions_path = '/versions'
|
||||
|
||||
# test for expand prefix, e.g. 001_expand_new_fk_constraint.py
|
||||
repo_path = self.repos[upgrades.EXPAND_REPO].repo_path
|
||||
repo_path = self.repos[upgrades.EXPAND_BRANCH].repo_path
|
||||
expand_list = glob.glob(repo_path + versions_path + '/*.py')
|
||||
self.assertRepoFileNamePrefix(expand_list, 'expand')
|
||||
|
||||
# test for migrate prefix, e.g. 001_migrate_new_fk_constraint.py
|
||||
repo_path = self.repos[upgrades.DATA_MIGRATION_REPO].repo_path
|
||||
repo_path = self.repos[upgrades.DATA_MIGRATION_BRANCH].repo_path
|
||||
migrate_list = glob.glob(repo_path + versions_path + '/*.py')
|
||||
self.assertRepoFileNamePrefix(migrate_list, 'migrate')
|
||||
|
||||
# test for contract prefix, e.g. 001_contract_new_fk_constraint.py
|
||||
repo_path = self.repos[upgrades.CONTRACT_REPO].repo_path
|
||||
repo_path = self.repos[upgrades.CONTRACT_BRANCH].repo_path
|
||||
contract_list = glob.glob(repo_path + versions_path + '/*.py')
|
||||
self.assertRepoFileNamePrefix(contract_list, 'contract')
|
||||
|
||||
|
|
@ -530,9 +530,9 @@ class MigrationValidation(MigrateBase, unit.TestCase):
|
|||
self.migrate(upgrades.INITIAL_VERSION + 1)
|
||||
self.contract(upgrades.INITIAL_VERSION + 1)
|
||||
for version in (
|
||||
upgrades.get_db_version('expand_repo'),
|
||||
upgrades.get_db_version('data_migration_repo'),
|
||||
upgrades.get_db_version('contract_repo'),
|
||||
upgrades.get_db_version('expand'),
|
||||
upgrades.get_db_version('data_migration'),
|
||||
upgrades.get_db_version('contract'),
|
||||
):
|
||||
self.assertEqual(upgrades.INITIAL_VERSION + 1, version)
|
||||
|
||||
|
|
@ -560,7 +560,7 @@ class FullMigration(MigrateBase, unit.TestCase):
|
|||
|
||||
def test_db_sync_check(self):
|
||||
checker = cli.DbSync()
|
||||
latest_version = self.repos[upgrades.EXPAND_REPO].max_version
|
||||
latest_version = self.repos[upgrades.EXPAND_BRANCH].max_version
|
||||
|
||||
# If the expand repository doesn't exist yet, then we need to make sure
|
||||
# we advertise that `--expand` must be run first.
|
||||
|
|
|
|||
Loading…
Reference in New Issue