From dce38678fc252bd39d3e28e920e76668e47398cd Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Thu, 20 Jan 2022 17:40:10 +0000 Subject: [PATCH] sql: Prepare for alembic migration Nothing functional here. We simply switch from using "repo_name" terminology and move/rename some helper functions. Branches aren't really a SQLAlchemy-Migrate thing but it's close enough to do. Change-Id: I005d20ef21b6c8122be90e8afb38abd902fdfc6e Signed-off-by: Stephen Finucane --- keystone/cmd/cli.py | 6 +- keystone/common/sql/upgrades.py | 182 ++++++++++++------------ keystone/tests/unit/test_sql_upgrade.py | 54 +++---- 3 files changed, 121 insertions(+), 121 deletions(-) diff --git a/keystone/cmd/cli.py b/keystone/cmd/cli.py index fae129a965..1e866d76a0 100644 --- a/keystone/cmd/cli.py +++ b/keystone/cmd/cli.py @@ -277,7 +277,7 @@ class DbSync(BaseApp): def check_db_sync_status(cls): status = 0 try: - expand_version = upgrades.get_db_version(repo='expand_repo') + expand_version = upgrades.get_db_version(branch='expand') except db_exception.DBMigrationError: LOG.info( 'Your database is not currently under version ' @@ -288,12 +288,12 @@ class DbSync(BaseApp): try: migrate_version = upgrades.get_db_version( - repo='data_migration_repo') + branch='data_migration') except db_exception.DBMigrationError: migrate_version = 0 try: - contract_version = upgrades.get_db_version(repo='contract_repo') + contract_version = upgrades.get_db_version(branch='contract') except db_exception.DBMigrationError: contract_version = 0 diff --git a/keystone/common/sql/upgrades.py b/keystone/common/sql/upgrades.py index f0657e01ac..f463771f28 100644 --- a/keystone/common/sql/upgrades.py +++ b/keystone/common/sql/upgrades.py @@ -28,17 +28,17 @@ from keystone.i18n import _ INITIAL_VERSION = 72 LATEST_VERSION = 79 -EXPAND_REPO = 'expand_repo' -DATA_MIGRATION_REPO = 'data_migration_repo' -CONTRACT_REPO = 'contract_repo' +EXPAND_BRANCH = 'expand' +DATA_MIGRATION_BRANCH = 'data_migration' +CONTRACT_BRANCH = 'contract' -def _get_migrate_repo_path(repo_name): +def _get_migrate_repo_path(branch): abs_path = os.path.abspath( os.path.join( os.path.dirname(sql.__file__), 'legacy_migrations', - repo_name, + f'{branch}_repo', ) ) @@ -142,8 +142,18 @@ def _migrate_db_sync(engine, abs_path, version=None, init_version=0): return migrate_api.downgrade(engine, repository, version) -def _sync_repo(repo_name): - abs_path = _get_migrate_repo_path(repo_name) +def get_db_version(branch=EXPAND_BRANCH): + abs_path = _get_migrate_repo_path(branch) + with sql.session_for_read() as session: + return _migrate_db_version( + session.get_bind(), + abs_path, + INITIAL_VERSION, + ) + + +def _db_sync(branch): + abs_path = _get_migrate_repo_path(branch) with sql.session_for_write() as session: engine = session.get_bind() _migrate_db_sync( @@ -153,6 +163,80 @@ def _sync_repo(repo_name): ) +def _validate_upgrade_order(branch, target_repo_version=None): + """Validate the state of the migration repositories. + + This is run before allowing the db_sync command to execute. Ensure the + upgrade step and version specified by the operator remains consistent with + the upgrade process. I.e. expand's version is greater or equal to + migrate's, migrate's version is greater or equal to contract's. + + :param branch: The name of the repository that the user is trying to + upgrade. + :param target_repo_version: The version to upgrade the repo. Otherwise, the + version will be upgraded to the latest version + available. + """ + # Initialize a dict to have each key assigned a repo with their value being + # the repo that comes before. + db_sync_order = { + DATA_MIGRATION_BRANCH: EXPAND_BRANCH, + CONTRACT_BRANCH: DATA_MIGRATION_BRANCH, + } + + if branch == EXPAND_BRANCH: + return + + # find the latest version that the current command will upgrade to if there + # wasn't a version specified for upgrade. + if not target_repo_version: + abs_path = _get_migrate_repo_path(branch) + repo = _find_migrate_repo(abs_path) + target_repo_version = int(repo.latest) + + # get current version of the command that runs before the current command. + dependency_repo_version = get_db_version(branch=db_sync_order[branch]) + + if dependency_repo_version < target_repo_version: + raise db_exception.DBMigrationError( + 'You are attempting to upgrade %s ahead of %s. Please refer to ' + 'https://docs.openstack.org/keystone/latest/admin/' + 'identity-upgrading.html ' + 'to see the proper steps for rolling upgrades.' % ( + branch, db_sync_order[branch])) + + +def expand_schema(): + """Expand the database schema ahead of data migration. + + This is run manually by the keystone-manage command before the first + keystone node is migrated to the latest release. + """ + _validate_upgrade_order(EXPAND_BRANCH) + _db_sync(branch=EXPAND_BRANCH) + + +def migrate_data(): + """Migrate data to match the new schema. + + This is run manually by the keystone-manage command once the keystone + schema has been expanded for the new release. + """ + _validate_upgrade_order(DATA_MIGRATION_BRANCH) + _db_sync(branch=DATA_MIGRATION_BRANCH) + + +def contract_schema(): + """Contract the database. + + This is run manually by the keystone-manage command once the keystone + nodes have been upgraded to the latest release and will remove any old + tables/columns that are no longer required. + """ + _validate_upgrade_order(CONTRACT_BRANCH) + _db_sync(branch=CONTRACT_BRANCH) + + def offline_sync_database_to_version(version=None): """Perform and off-line sync of the database. @@ -171,87 +255,3 @@ def offline_sync_database_to_version(version=None): expand_schema() migrate_data() contract_schema() - - -def get_db_version(repo=EXPAND_REPO): - abs_path = _get_migrate_repo_path(repo) - with sql.session_for_read() as session: - return _migrate_db_version( - session.get_bind(), - abs_path, - INITIAL_VERSION, - ) - - -def validate_upgrade_order(repo_name, target_repo_version=None): - """Validate the state of the migration repositories. - - This is run before allowing the db_sync command to execute. Ensure the - upgrade step and version specified by the operator remains consistent with - the upgrade process. I.e. expand's version is greater or equal to - migrate's, migrate's version is greater or equal to contract's. - - :param repo_name: The name of the repository that the user is trying to - upgrade. - :param target_repo_version: The version to upgrade the repo. Otherwise, the - version will be upgraded to the latest version - available. - """ - # Initialize a dict to have each key assigned a repo with their value being - # the repo that comes before. - db_sync_order = { - DATA_MIGRATION_REPO: EXPAND_REPO, - CONTRACT_REPO: DATA_MIGRATION_REPO, - } - - if repo_name == EXPAND_REPO: - return - - # find the latest version that the current command will upgrade to if there - # wasn't a version specified for upgrade. - if not target_repo_version: - abs_path = _get_migrate_repo_path(repo_name) - repo = _find_migrate_repo(abs_path) - target_repo_version = int(repo.latest) - - # get current version of the command that runs before the current command. - dependency_repo_version = get_db_version(repo=db_sync_order[repo_name]) - - if dependency_repo_version < target_repo_version: - raise db_exception.DBMigrationError( - 'You are attempting to upgrade %s ahead of %s. Please refer to ' - 'https://docs.openstack.org/keystone/latest/admin/' - 'identity-upgrading.html ' - 'to see the proper steps for rolling upgrades.' % ( - repo_name, db_sync_order[repo_name])) - - -def expand_schema(): - """Expand the database schema ahead of data migration. - - This is run manually by the keystone-manage command before the first - keystone node is migrated to the latest release. - """ - validate_upgrade_order(EXPAND_REPO) - _sync_repo(repo_name=EXPAND_REPO) - - -def migrate_data(): - """Migrate data to match the new schema. - - This is run manually by the keystone-manage command once the keystone - schema has been expanded for the new release. - """ - validate_upgrade_order(DATA_MIGRATION_REPO) - _sync_repo(repo_name=DATA_MIGRATION_REPO) - - -def contract_schema(): - """Contract the database. - - This is run manually by the keystone-manage command once the keystone - nodes have been upgraded to the latest release and will remove any old - tables/columns that are no longer required. - """ - validate_upgrade_order(CONTRACT_REPO) - _sync_repo(repo_name=CONTRACT_REPO) diff --git a/keystone/tests/unit/test_sql_upgrade.py b/keystone/tests/unit/test_sql_upgrade.py index 12871d40b0..78f6449778 100644 --- a/keystone/tests/unit/test_sql_upgrade.py +++ b/keystone/tests/unit/test_sql_upgrade.py @@ -248,7 +248,7 @@ class Repository: version = migrate_api._migrate_version( self.schema_, version, upgrade, err, ) - upgrades.validate_upgrade_order( + upgrades._validate_upgrade_order( self.repo_name, target_repo_version=version, ) if not current_schema: @@ -305,28 +305,28 @@ class MigrateBase( self.addCleanup(sql.cleanup) self.repos = { - upgrades.EXPAND_REPO: Repository( - self.engine, upgrades.EXPAND_REPO, + upgrades.EXPAND_BRANCH: Repository( + self.engine, upgrades.EXPAND_BRANCH, ), - upgrades.DATA_MIGRATION_REPO: Repository( - self.engine, upgrades.DATA_MIGRATION_REPO, + upgrades.DATA_MIGRATION_BRANCH: Repository( + self.engine, upgrades.DATA_MIGRATION_BRANCH, ), - upgrades.CONTRACT_REPO: Repository( - self.engine, upgrades.CONTRACT_REPO, + upgrades.CONTRACT_BRANCH: Repository( + self.engine, upgrades.CONTRACT_BRANCH, ), } def expand(self, *args, **kwargs): """Expand database schema.""" - self.repos[upgrades.EXPAND_REPO].upgrade(*args, **kwargs) + self.repos[upgrades.EXPAND_BRANCH].upgrade(*args, **kwargs) def migrate(self, *args, **kwargs): """Migrate data.""" - self.repos[upgrades.DATA_MIGRATION_REPO].upgrade(*args, **kwargs) + self.repos[upgrades.DATA_MIGRATION_BRANCH].upgrade(*args, **kwargs) def contract(self, *args, **kwargs): """Contract database schema.""" - self.repos[upgrades.CONTRACT_REPO].upgrade(*args, **kwargs) + self.repos[upgrades.CONTRACT_BRANCH].upgrade(*args, **kwargs) @property def metadata(self): @@ -362,8 +362,8 @@ class ExpandSchemaUpgradeTests(MigrateBase): def test_start_version_db_init_version(self): self.assertEqual( - self.repos[upgrades.EXPAND_REPO].min_version, - self.repos[upgrades.EXPAND_REPO].version) + self.repos[upgrades.EXPAND_BRANCH].min_version, + self.repos[upgrades.EXPAND_BRANCH].version) def test_blank_db_to_start(self): self.assertTableDoesNotExist('user') @@ -399,8 +399,8 @@ class DataMigrationUpgradeTests(MigrateBase): def test_start_version_db_init_version(self): self.assertEqual( - self.repos[upgrades.DATA_MIGRATION_REPO].min_version, - self.repos[upgrades.DATA_MIGRATION_REPO].version, + self.repos[upgrades.DATA_MIGRATION_BRANCH].min_version, + self.repos[upgrades.DATA_MIGRATION_BRANCH].version, ) @@ -435,8 +435,8 @@ class ContractSchemaUpgradeTests(MigrateBase, unit.TestCase): def test_start_version_db_init_version(self): self.assertEqual( - self.repos[upgrades.CONTRACT_REPO].min_version, - self.repos[upgrades.CONTRACT_REPO].version, + self.repos[upgrades.CONTRACT_BRANCH].min_version, + self.repos[upgrades.CONTRACT_BRANCH].version, ) @@ -476,12 +476,12 @@ class VersionTests(MigrateBase): """ # Transitive comparison: expand == data migration == contract self.assertEqual( - self.repos[upgrades.EXPAND_REPO].max_version, - self.repos[upgrades.DATA_MIGRATION_REPO].max_version, + self.repos[upgrades.EXPAND_BRANCH].max_version, + self.repos[upgrades.DATA_MIGRATION_BRANCH].max_version, ) self.assertEqual( - self.repos[upgrades.DATA_MIGRATION_REPO].max_version, - self.repos[upgrades.CONTRACT_REPO].max_version, + self.repos[upgrades.DATA_MIGRATION_BRANCH].max_version, + self.repos[upgrades.CONTRACT_BRANCH].max_version, ) def test_migrate_repos_file_names_have_prefix(self): @@ -496,17 +496,17 @@ class VersionTests(MigrateBase): versions_path = '/versions' # test for expand prefix, e.g. 001_expand_new_fk_constraint.py - repo_path = self.repos[upgrades.EXPAND_REPO].repo_path + repo_path = self.repos[upgrades.EXPAND_BRANCH].repo_path expand_list = glob.glob(repo_path + versions_path + '/*.py') self.assertRepoFileNamePrefix(expand_list, 'expand') # test for migrate prefix, e.g. 001_migrate_new_fk_constraint.py - repo_path = self.repos[upgrades.DATA_MIGRATION_REPO].repo_path + repo_path = self.repos[upgrades.DATA_MIGRATION_BRANCH].repo_path migrate_list = glob.glob(repo_path + versions_path + '/*.py') self.assertRepoFileNamePrefix(migrate_list, 'migrate') # test for contract prefix, e.g. 001_contract_new_fk_constraint.py - repo_path = self.repos[upgrades.CONTRACT_REPO].repo_path + repo_path = self.repos[upgrades.CONTRACT_BRANCH].repo_path contract_list = glob.glob(repo_path + versions_path + '/*.py') self.assertRepoFileNamePrefix(contract_list, 'contract') @@ -530,9 +530,9 @@ class MigrationValidation(MigrateBase, unit.TestCase): self.migrate(upgrades.INITIAL_VERSION + 1) self.contract(upgrades.INITIAL_VERSION + 1) for version in ( - upgrades.get_db_version('expand_repo'), - upgrades.get_db_version('data_migration_repo'), - upgrades.get_db_version('contract_repo'), + upgrades.get_db_version('expand'), + upgrades.get_db_version('data_migration'), + upgrades.get_db_version('contract'), ): self.assertEqual(upgrades.INITIAL_VERSION + 1, version) @@ -560,7 +560,7 @@ class FullMigration(MigrateBase, unit.TestCase): def test_db_sync_check(self): checker = cli.DbSync() - latest_version = self.repos[upgrades.EXPAND_REPO].max_version + latest_version = self.repos[upgrades.EXPAND_BRANCH].max_version # If the expand repository doesn't exist yet, then we need to make sure # we advertise that `--expand` must be run first.