sql: Prepare for alembic migration
Nothing functional here. We simply switch from using "repo_name" terminology and move/rename some helper functions. Branches aren't really a SQLAlchemy-Migrate thing but it's close enough to do. Change-Id: I005d20ef21b6c8122be90e8afb38abd902fdfc6e Signed-off-by: Stephen Finucane <stephenfin@redhat.com>
This commit is contained in:
parent
a6887c13ea
commit
dce38678fc
|
@ -277,7 +277,7 @@ class DbSync(BaseApp):
|
||||||
def check_db_sync_status(cls):
|
def check_db_sync_status(cls):
|
||||||
status = 0
|
status = 0
|
||||||
try:
|
try:
|
||||||
expand_version = upgrades.get_db_version(repo='expand_repo')
|
expand_version = upgrades.get_db_version(branch='expand')
|
||||||
except db_exception.DBMigrationError:
|
except db_exception.DBMigrationError:
|
||||||
LOG.info(
|
LOG.info(
|
||||||
'Your database is not currently under version '
|
'Your database is not currently under version '
|
||||||
|
@ -288,12 +288,12 @@ class DbSync(BaseApp):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
migrate_version = upgrades.get_db_version(
|
migrate_version = upgrades.get_db_version(
|
||||||
repo='data_migration_repo')
|
branch='data_migration')
|
||||||
except db_exception.DBMigrationError:
|
except db_exception.DBMigrationError:
|
||||||
migrate_version = 0
|
migrate_version = 0
|
||||||
|
|
||||||
try:
|
try:
|
||||||
contract_version = upgrades.get_db_version(repo='contract_repo')
|
contract_version = upgrades.get_db_version(branch='contract')
|
||||||
except db_exception.DBMigrationError:
|
except db_exception.DBMigrationError:
|
||||||
contract_version = 0
|
contract_version = 0
|
||||||
|
|
||||||
|
|
|
@ -28,17 +28,17 @@ from keystone.i18n import _
|
||||||
|
|
||||||
INITIAL_VERSION = 72
|
INITIAL_VERSION = 72
|
||||||
LATEST_VERSION = 79
|
LATEST_VERSION = 79
|
||||||
EXPAND_REPO = 'expand_repo'
|
EXPAND_BRANCH = 'expand'
|
||||||
DATA_MIGRATION_REPO = 'data_migration_repo'
|
DATA_MIGRATION_BRANCH = 'data_migration'
|
||||||
CONTRACT_REPO = 'contract_repo'
|
CONTRACT_BRANCH = 'contract'
|
||||||
|
|
||||||
|
|
||||||
def _get_migrate_repo_path(repo_name):
|
def _get_migrate_repo_path(branch):
|
||||||
abs_path = os.path.abspath(
|
abs_path = os.path.abspath(
|
||||||
os.path.join(
|
os.path.join(
|
||||||
os.path.dirname(sql.__file__),
|
os.path.dirname(sql.__file__),
|
||||||
'legacy_migrations',
|
'legacy_migrations',
|
||||||
repo_name,
|
f'{branch}_repo',
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -142,8 +142,18 @@ def _migrate_db_sync(engine, abs_path, version=None, init_version=0):
|
||||||
return migrate_api.downgrade(engine, repository, version)
|
return migrate_api.downgrade(engine, repository, version)
|
||||||
|
|
||||||
|
|
||||||
def _sync_repo(repo_name):
|
def get_db_version(branch=EXPAND_BRANCH):
|
||||||
abs_path = _get_migrate_repo_path(repo_name)
|
abs_path = _get_migrate_repo_path(branch)
|
||||||
|
with sql.session_for_read() as session:
|
||||||
|
return _migrate_db_version(
|
||||||
|
session.get_bind(),
|
||||||
|
abs_path,
|
||||||
|
INITIAL_VERSION,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _db_sync(branch):
|
||||||
|
abs_path = _get_migrate_repo_path(branch)
|
||||||
with sql.session_for_write() as session:
|
with sql.session_for_write() as session:
|
||||||
engine = session.get_bind()
|
engine = session.get_bind()
|
||||||
_migrate_db_sync(
|
_migrate_db_sync(
|
||||||
|
@ -153,6 +163,80 @@ def _sync_repo(repo_name):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_upgrade_order(branch, target_repo_version=None):
|
||||||
|
"""Validate the state of the migration repositories.
|
||||||
|
|
||||||
|
This is run before allowing the db_sync command to execute. Ensure the
|
||||||
|
upgrade step and version specified by the operator remains consistent with
|
||||||
|
the upgrade process. I.e. expand's version is greater or equal to
|
||||||
|
migrate's, migrate's version is greater or equal to contract's.
|
||||||
|
|
||||||
|
:param branch: The name of the repository that the user is trying to
|
||||||
|
upgrade.
|
||||||
|
:param target_repo_version: The version to upgrade the repo. Otherwise, the
|
||||||
|
version will be upgraded to the latest version
|
||||||
|
available.
|
||||||
|
"""
|
||||||
|
# Initialize a dict to have each key assigned a repo with their value being
|
||||||
|
# the repo that comes before.
|
||||||
|
db_sync_order = {
|
||||||
|
DATA_MIGRATION_BRANCH: EXPAND_BRANCH,
|
||||||
|
CONTRACT_BRANCH: DATA_MIGRATION_BRANCH,
|
||||||
|
}
|
||||||
|
|
||||||
|
if branch == EXPAND_BRANCH:
|
||||||
|
return
|
||||||
|
|
||||||
|
# find the latest version that the current command will upgrade to if there
|
||||||
|
# wasn't a version specified for upgrade.
|
||||||
|
if not target_repo_version:
|
||||||
|
abs_path = _get_migrate_repo_path(branch)
|
||||||
|
repo = _find_migrate_repo(abs_path)
|
||||||
|
target_repo_version = int(repo.latest)
|
||||||
|
|
||||||
|
# get current version of the command that runs before the current command.
|
||||||
|
dependency_repo_version = get_db_version(branch=db_sync_order[branch])
|
||||||
|
|
||||||
|
if dependency_repo_version < target_repo_version:
|
||||||
|
raise db_exception.DBMigrationError(
|
||||||
|
'You are attempting to upgrade %s ahead of %s. Please refer to '
|
||||||
|
'https://docs.openstack.org/keystone/latest/admin/'
|
||||||
|
'identity-upgrading.html '
|
||||||
|
'to see the proper steps for rolling upgrades.' % (
|
||||||
|
branch, db_sync_order[branch]))
|
||||||
|
|
||||||
|
|
||||||
|
def expand_schema():
|
||||||
|
"""Expand the database schema ahead of data migration.
|
||||||
|
|
||||||
|
This is run manually by the keystone-manage command before the first
|
||||||
|
keystone node is migrated to the latest release.
|
||||||
|
"""
|
||||||
|
_validate_upgrade_order(EXPAND_BRANCH)
|
||||||
|
_db_sync(branch=EXPAND_BRANCH)
|
||||||
|
|
||||||
|
|
||||||
|
def migrate_data():
|
||||||
|
"""Migrate data to match the new schema.
|
||||||
|
|
||||||
|
This is run manually by the keystone-manage command once the keystone
|
||||||
|
schema has been expanded for the new release.
|
||||||
|
"""
|
||||||
|
_validate_upgrade_order(DATA_MIGRATION_BRANCH)
|
||||||
|
_db_sync(branch=DATA_MIGRATION_BRANCH)
|
||||||
|
|
||||||
|
|
||||||
|
def contract_schema():
|
||||||
|
"""Contract the database.
|
||||||
|
|
||||||
|
This is run manually by the keystone-manage command once the keystone
|
||||||
|
nodes have been upgraded to the latest release and will remove any old
|
||||||
|
tables/columns that are no longer required.
|
||||||
|
"""
|
||||||
|
_validate_upgrade_order(CONTRACT_BRANCH)
|
||||||
|
_db_sync(branch=CONTRACT_BRANCH)
|
||||||
|
|
||||||
|
|
||||||
def offline_sync_database_to_version(version=None):
|
def offline_sync_database_to_version(version=None):
|
||||||
"""Perform and off-line sync of the database.
|
"""Perform and off-line sync of the database.
|
||||||
|
|
||||||
|
@ -171,87 +255,3 @@ def offline_sync_database_to_version(version=None):
|
||||||
expand_schema()
|
expand_schema()
|
||||||
migrate_data()
|
migrate_data()
|
||||||
contract_schema()
|
contract_schema()
|
||||||
|
|
||||||
|
|
||||||
def get_db_version(repo=EXPAND_REPO):
|
|
||||||
abs_path = _get_migrate_repo_path(repo)
|
|
||||||
with sql.session_for_read() as session:
|
|
||||||
return _migrate_db_version(
|
|
||||||
session.get_bind(),
|
|
||||||
abs_path,
|
|
||||||
INITIAL_VERSION,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def validate_upgrade_order(repo_name, target_repo_version=None):
|
|
||||||
"""Validate the state of the migration repositories.
|
|
||||||
|
|
||||||
This is run before allowing the db_sync command to execute. Ensure the
|
|
||||||
upgrade step and version specified by the operator remains consistent with
|
|
||||||
the upgrade process. I.e. expand's version is greater or equal to
|
|
||||||
migrate's, migrate's version is greater or equal to contract's.
|
|
||||||
|
|
||||||
:param repo_name: The name of the repository that the user is trying to
|
|
||||||
upgrade.
|
|
||||||
:param target_repo_version: The version to upgrade the repo. Otherwise, the
|
|
||||||
version will be upgraded to the latest version
|
|
||||||
available.
|
|
||||||
"""
|
|
||||||
# Initialize a dict to have each key assigned a repo with their value being
|
|
||||||
# the repo that comes before.
|
|
||||||
db_sync_order = {
|
|
||||||
DATA_MIGRATION_REPO: EXPAND_REPO,
|
|
||||||
CONTRACT_REPO: DATA_MIGRATION_REPO,
|
|
||||||
}
|
|
||||||
|
|
||||||
if repo_name == EXPAND_REPO:
|
|
||||||
return
|
|
||||||
|
|
||||||
# find the latest version that the current command will upgrade to if there
|
|
||||||
# wasn't a version specified for upgrade.
|
|
||||||
if not target_repo_version:
|
|
||||||
abs_path = _get_migrate_repo_path(repo_name)
|
|
||||||
repo = _find_migrate_repo(abs_path)
|
|
||||||
target_repo_version = int(repo.latest)
|
|
||||||
|
|
||||||
# get current version of the command that runs before the current command.
|
|
||||||
dependency_repo_version = get_db_version(repo=db_sync_order[repo_name])
|
|
||||||
|
|
||||||
if dependency_repo_version < target_repo_version:
|
|
||||||
raise db_exception.DBMigrationError(
|
|
||||||
'You are attempting to upgrade %s ahead of %s. Please refer to '
|
|
||||||
'https://docs.openstack.org/keystone/latest/admin/'
|
|
||||||
'identity-upgrading.html '
|
|
||||||
'to see the proper steps for rolling upgrades.' % (
|
|
||||||
repo_name, db_sync_order[repo_name]))
|
|
||||||
|
|
||||||
|
|
||||||
def expand_schema():
|
|
||||||
"""Expand the database schema ahead of data migration.
|
|
||||||
|
|
||||||
This is run manually by the keystone-manage command before the first
|
|
||||||
keystone node is migrated to the latest release.
|
|
||||||
"""
|
|
||||||
validate_upgrade_order(EXPAND_REPO)
|
|
||||||
_sync_repo(repo_name=EXPAND_REPO)
|
|
||||||
|
|
||||||
|
|
||||||
def migrate_data():
|
|
||||||
"""Migrate data to match the new schema.
|
|
||||||
|
|
||||||
This is run manually by the keystone-manage command once the keystone
|
|
||||||
schema has been expanded for the new release.
|
|
||||||
"""
|
|
||||||
validate_upgrade_order(DATA_MIGRATION_REPO)
|
|
||||||
_sync_repo(repo_name=DATA_MIGRATION_REPO)
|
|
||||||
|
|
||||||
|
|
||||||
def contract_schema():
|
|
||||||
"""Contract the database.
|
|
||||||
|
|
||||||
This is run manually by the keystone-manage command once the keystone
|
|
||||||
nodes have been upgraded to the latest release and will remove any old
|
|
||||||
tables/columns that are no longer required.
|
|
||||||
"""
|
|
||||||
validate_upgrade_order(CONTRACT_REPO)
|
|
||||||
_sync_repo(repo_name=CONTRACT_REPO)
|
|
||||||
|
|
|
@ -248,7 +248,7 @@ class Repository:
|
||||||
version = migrate_api._migrate_version(
|
version = migrate_api._migrate_version(
|
||||||
self.schema_, version, upgrade, err,
|
self.schema_, version, upgrade, err,
|
||||||
)
|
)
|
||||||
upgrades.validate_upgrade_order(
|
upgrades._validate_upgrade_order(
|
||||||
self.repo_name, target_repo_version=version,
|
self.repo_name, target_repo_version=version,
|
||||||
)
|
)
|
||||||
if not current_schema:
|
if not current_schema:
|
||||||
|
@ -305,28 +305,28 @@ class MigrateBase(
|
||||||
self.addCleanup(sql.cleanup)
|
self.addCleanup(sql.cleanup)
|
||||||
|
|
||||||
self.repos = {
|
self.repos = {
|
||||||
upgrades.EXPAND_REPO: Repository(
|
upgrades.EXPAND_BRANCH: Repository(
|
||||||
self.engine, upgrades.EXPAND_REPO,
|
self.engine, upgrades.EXPAND_BRANCH,
|
||||||
),
|
),
|
||||||
upgrades.DATA_MIGRATION_REPO: Repository(
|
upgrades.DATA_MIGRATION_BRANCH: Repository(
|
||||||
self.engine, upgrades.DATA_MIGRATION_REPO,
|
self.engine, upgrades.DATA_MIGRATION_BRANCH,
|
||||||
),
|
),
|
||||||
upgrades.CONTRACT_REPO: Repository(
|
upgrades.CONTRACT_BRANCH: Repository(
|
||||||
self.engine, upgrades.CONTRACT_REPO,
|
self.engine, upgrades.CONTRACT_BRANCH,
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
def expand(self, *args, **kwargs):
|
def expand(self, *args, **kwargs):
|
||||||
"""Expand database schema."""
|
"""Expand database schema."""
|
||||||
self.repos[upgrades.EXPAND_REPO].upgrade(*args, **kwargs)
|
self.repos[upgrades.EXPAND_BRANCH].upgrade(*args, **kwargs)
|
||||||
|
|
||||||
def migrate(self, *args, **kwargs):
|
def migrate(self, *args, **kwargs):
|
||||||
"""Migrate data."""
|
"""Migrate data."""
|
||||||
self.repos[upgrades.DATA_MIGRATION_REPO].upgrade(*args, **kwargs)
|
self.repos[upgrades.DATA_MIGRATION_BRANCH].upgrade(*args, **kwargs)
|
||||||
|
|
||||||
def contract(self, *args, **kwargs):
|
def contract(self, *args, **kwargs):
|
||||||
"""Contract database schema."""
|
"""Contract database schema."""
|
||||||
self.repos[upgrades.CONTRACT_REPO].upgrade(*args, **kwargs)
|
self.repos[upgrades.CONTRACT_BRANCH].upgrade(*args, **kwargs)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def metadata(self):
|
def metadata(self):
|
||||||
|
@ -362,8 +362,8 @@ class ExpandSchemaUpgradeTests(MigrateBase):
|
||||||
|
|
||||||
def test_start_version_db_init_version(self):
|
def test_start_version_db_init_version(self):
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self.repos[upgrades.EXPAND_REPO].min_version,
|
self.repos[upgrades.EXPAND_BRANCH].min_version,
|
||||||
self.repos[upgrades.EXPAND_REPO].version)
|
self.repos[upgrades.EXPAND_BRANCH].version)
|
||||||
|
|
||||||
def test_blank_db_to_start(self):
|
def test_blank_db_to_start(self):
|
||||||
self.assertTableDoesNotExist('user')
|
self.assertTableDoesNotExist('user')
|
||||||
|
@ -399,8 +399,8 @@ class DataMigrationUpgradeTests(MigrateBase):
|
||||||
|
|
||||||
def test_start_version_db_init_version(self):
|
def test_start_version_db_init_version(self):
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self.repos[upgrades.DATA_MIGRATION_REPO].min_version,
|
self.repos[upgrades.DATA_MIGRATION_BRANCH].min_version,
|
||||||
self.repos[upgrades.DATA_MIGRATION_REPO].version,
|
self.repos[upgrades.DATA_MIGRATION_BRANCH].version,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -435,8 +435,8 @@ class ContractSchemaUpgradeTests(MigrateBase, unit.TestCase):
|
||||||
|
|
||||||
def test_start_version_db_init_version(self):
|
def test_start_version_db_init_version(self):
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self.repos[upgrades.CONTRACT_REPO].min_version,
|
self.repos[upgrades.CONTRACT_BRANCH].min_version,
|
||||||
self.repos[upgrades.CONTRACT_REPO].version,
|
self.repos[upgrades.CONTRACT_BRANCH].version,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -476,12 +476,12 @@ class VersionTests(MigrateBase):
|
||||||
"""
|
"""
|
||||||
# Transitive comparison: expand == data migration == contract
|
# Transitive comparison: expand == data migration == contract
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self.repos[upgrades.EXPAND_REPO].max_version,
|
self.repos[upgrades.EXPAND_BRANCH].max_version,
|
||||||
self.repos[upgrades.DATA_MIGRATION_REPO].max_version,
|
self.repos[upgrades.DATA_MIGRATION_BRANCH].max_version,
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self.repos[upgrades.DATA_MIGRATION_REPO].max_version,
|
self.repos[upgrades.DATA_MIGRATION_BRANCH].max_version,
|
||||||
self.repos[upgrades.CONTRACT_REPO].max_version,
|
self.repos[upgrades.CONTRACT_BRANCH].max_version,
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_migrate_repos_file_names_have_prefix(self):
|
def test_migrate_repos_file_names_have_prefix(self):
|
||||||
|
@ -496,17 +496,17 @@ class VersionTests(MigrateBase):
|
||||||
versions_path = '/versions'
|
versions_path = '/versions'
|
||||||
|
|
||||||
# test for expand prefix, e.g. 001_expand_new_fk_constraint.py
|
# test for expand prefix, e.g. 001_expand_new_fk_constraint.py
|
||||||
repo_path = self.repos[upgrades.EXPAND_REPO].repo_path
|
repo_path = self.repos[upgrades.EXPAND_BRANCH].repo_path
|
||||||
expand_list = glob.glob(repo_path + versions_path + '/*.py')
|
expand_list = glob.glob(repo_path + versions_path + '/*.py')
|
||||||
self.assertRepoFileNamePrefix(expand_list, 'expand')
|
self.assertRepoFileNamePrefix(expand_list, 'expand')
|
||||||
|
|
||||||
# test for migrate prefix, e.g. 001_migrate_new_fk_constraint.py
|
# test for migrate prefix, e.g. 001_migrate_new_fk_constraint.py
|
||||||
repo_path = self.repos[upgrades.DATA_MIGRATION_REPO].repo_path
|
repo_path = self.repos[upgrades.DATA_MIGRATION_BRANCH].repo_path
|
||||||
migrate_list = glob.glob(repo_path + versions_path + '/*.py')
|
migrate_list = glob.glob(repo_path + versions_path + '/*.py')
|
||||||
self.assertRepoFileNamePrefix(migrate_list, 'migrate')
|
self.assertRepoFileNamePrefix(migrate_list, 'migrate')
|
||||||
|
|
||||||
# test for contract prefix, e.g. 001_contract_new_fk_constraint.py
|
# test for contract prefix, e.g. 001_contract_new_fk_constraint.py
|
||||||
repo_path = self.repos[upgrades.CONTRACT_REPO].repo_path
|
repo_path = self.repos[upgrades.CONTRACT_BRANCH].repo_path
|
||||||
contract_list = glob.glob(repo_path + versions_path + '/*.py')
|
contract_list = glob.glob(repo_path + versions_path + '/*.py')
|
||||||
self.assertRepoFileNamePrefix(contract_list, 'contract')
|
self.assertRepoFileNamePrefix(contract_list, 'contract')
|
||||||
|
|
||||||
|
@ -530,9 +530,9 @@ class MigrationValidation(MigrateBase, unit.TestCase):
|
||||||
self.migrate(upgrades.INITIAL_VERSION + 1)
|
self.migrate(upgrades.INITIAL_VERSION + 1)
|
||||||
self.contract(upgrades.INITIAL_VERSION + 1)
|
self.contract(upgrades.INITIAL_VERSION + 1)
|
||||||
for version in (
|
for version in (
|
||||||
upgrades.get_db_version('expand_repo'),
|
upgrades.get_db_version('expand'),
|
||||||
upgrades.get_db_version('data_migration_repo'),
|
upgrades.get_db_version('data_migration'),
|
||||||
upgrades.get_db_version('contract_repo'),
|
upgrades.get_db_version('contract'),
|
||||||
):
|
):
|
||||||
self.assertEqual(upgrades.INITIAL_VERSION + 1, version)
|
self.assertEqual(upgrades.INITIAL_VERSION + 1, version)
|
||||||
|
|
||||||
|
@ -560,7 +560,7 @@ class FullMigration(MigrateBase, unit.TestCase):
|
||||||
|
|
||||||
def test_db_sync_check(self):
|
def test_db_sync_check(self):
|
||||||
checker = cli.DbSync()
|
checker = cli.DbSync()
|
||||||
latest_version = self.repos[upgrades.EXPAND_REPO].max_version
|
latest_version = self.repos[upgrades.EXPAND_BRANCH].max_version
|
||||||
|
|
||||||
# If the expand repository doesn't exist yet, then we need to make sure
|
# If the expand repository doesn't exist yet, then we need to make sure
|
||||||
# we advertise that `--expand` must be run first.
|
# we advertise that `--expand` must be run first.
|
||||||
|
|
Loading…
Reference in New Issue