From 4569d41e13aa6196311de46cd4dab3449dcac11f Mon Sep 17 00:00:00 2001 From: Henry Nash Date: Tue, 5 Jul 2016 10:10:11 +0100 Subject: [PATCH] Add support for rolling upgrades to keystone-manage Add all the commands to keystone-manage as well as stubs for the logic in the migration helpers, for the expand, migrate and contract cycles of a rolling upgrade. Follow-on patchs will add the logic to the migration helpers. Partially Implements blueprint manage-migration Change-Id: I9f138fe0bcbf5ffbb98e6fcebd7d897329a301b7 --- keystone/cmd/cli.py | 32 +++++++++++- keystone/common/sql/migration_helpers.py | 54 ++++++++++++++++++++- keystone/tests/unit/test_cli.py | 62 ++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 3 deletions(-) diff --git a/keystone/cmd/cli.py b/keystone/cmd/cli.py index 3ebe2f661c..0eb41a998d 100644 --- a/keystone/cmd/cli.py +++ b/keystone/cmd/cli.py @@ -407,14 +407,42 @@ class DbSync(BaseApp): 'now part of the main repository, ' 'specifying db_sync without this option ' 'will cause all extensions to be migrated.')) + group = parser.add_mutually_exclusive_group() + group.add_argument('--expand', default=False, action='store_true', + help=('Expand the database schema in preparation ' + 'for data migration and starting the first ' + 'keystone node upgraded to the new release.')) + group.add_argument('--migrate', default=False, + action='store_true', + help=('Copy all data that needs to be migrated ' + 'within the database ahead of starting the ' + 'first keystone node upgraded to the new ' + 'release. This command should be run ' + 'after the --expand command. Once the ' + '--migrate command has completed, you can ' + 'upgrade all your keystone nodes to the new ' + 'release and restart them.')) + group.add_argument('--contract', default=False, action='store_true', + help=('Remove any database tables and columns ' + 'that are no longer required. This command ' + 'should be run after all keystone nodes are ' + 'running the new release.')) return parser @staticmethod def main(): assert_not_extension(CONF.command.extension) - version = CONF.command.version - migration_helpers.sync_database_to_version(version) + + if CONF.command.expand: + migration_helpers.expand_schema() + elif CONF.command.migrate: + migration_helpers.migrate_data() + elif CONF.command.contract: + migration_helpers.contract_schema() + else: + migration_helpers.offline_sync_database_to_version( + CONF.command.version) class DbVersion(BaseApp): diff --git a/keystone/common/sql/migration_helpers.py b/keystone/common/sql/migration_helpers.py index 07a0e2eb03..d4f3e68b9f 100644 --- a/keystone/common/sql/migration_helpers.py +++ b/keystone/common/sql/migration_helpers.py @@ -160,9 +160,25 @@ def _assert_not_schema_downgrade(version=None): pass -def sync_database_to_version(version=None): +def offline_sync_database_to_version(version=None): + """Perform and off-line sync of the database. + + Migrate the database up to the latest version, doing the equivalent of + the cycle of --expand, --migrate and --contract, for when an offline + upgrade is being performed. + + If a version is specified then only migrate the database up to that + version. Downgrading is not supported. If version is specified, then only + the main database migration is carried out - and the data migration and + contract phases will NOT be run. + + """ _sync_common_repo(version) + if not version: + migrate_data() + contract_schema() + def get_db_version(): with sql.session_for_write() as session: @@ -173,3 +189,39 @@ def get_db_version(): def print_db_version(): print(get_db_version()) + + +def expand_schema(): + """Expand the database schema ahead of data migration. + + This is run manually by the keystone-manage command before the first + keystone node is migrated to the latest release. + + """ + # TODO(henry-nash): Add implementation here. + pass + + +def migrate_data(): + """Migrate data to match the new schema. + + This is run manually by the keystone-manage command once the keystone + schema has been expanded for the new release. + + """ + # TODO(henry-nash): Add implementation here. + pass + + +def contract_schema(): + """Contract the database. + + This is run manually by the keystone-manage command once the keystone + nodes have been upgraded to the latest release and will remove any old + tables/columns that are no longer required. In addition, if any data + could have been left inconsistent while running with a mix of releases, + then this should be fixed up here. + + """ + # TODO(henry-nash): Add implementation here. + pass diff --git a/keystone/tests/unit/test_cli.py b/keystone/tests/unit/test_cli.py index 14f6d07ccc..f412fbd39b 100644 --- a/keystone/tests/unit/test_cli.py +++ b/keystone/tests/unit/test_cli.py @@ -25,6 +25,7 @@ from testtools import matchers from keystone.cmd import cli from keystone.common import dependency +from keystone.common.sql import migration_helpers import keystone.conf from keystone.i18n import _ from keystone.tests import unit @@ -534,3 +535,64 @@ class TestDomainConfigFinder(unit.BaseTestCase): self.assertThat( self.logging.output, matchers.Contains(expected_msg_template % 'keystone.conf')) + + +class CliDBSyncTestCase(unit.BaseTestCase): + + class FakeConfCommand(object): + def __init__(self, parent): + self.extension = False + self.expand = parent.command_expand + self.migrate = parent.command_migrate + self.contract = parent.command_contract + self.version = None + + def setUp(self): + super(CliDBSyncTestCase, self).setUp() + self.config_fixture = self.useFixture(config_fixture.Config(CONF)) + self.config_fixture.register_cli_opt(cli.command_opt) + migration_helpers.offline_sync_database_to_version = mock.Mock() + migration_helpers.expand_schema = mock.Mock() + migration_helpers.migrate_data = mock.Mock() + migration_helpers.contract_schema = mock.Mock() + self.command_expand = False + self.command_migrate = False + self.command_contract = False + + def _assert_correct_call(self, mocked_function): + for func in [migration_helpers.offline_sync_database_to_version, + migration_helpers.expand_schema, + migration_helpers.migrate_data, + migration_helpers.contract_schema]: + if func == mocked_function: + self.assertTrue(func.called) + else: + self.assertFalse(func.called) + + def test_db_sync(self): + self.useFixture(mockpatch.PatchObject( + CONF, 'command', self.FakeConfCommand(self))) + cli.DbSync.main() + self._assert_correct_call( + migration_helpers.offline_sync_database_to_version) + + def test_db_sync_expand(self): + self.command_expand = True + self.useFixture(mockpatch.PatchObject( + CONF, 'command', self.FakeConfCommand(self))) + cli.DbSync.main() + self._assert_correct_call(migration_helpers.expand_schema) + + def test_db_sync_migrate(self): + self.command_migrate = True + self.useFixture(mockpatch.PatchObject( + CONF, 'command', self.FakeConfCommand(self))) + cli.DbSync.main() + self._assert_correct_call(migration_helpers.migrate_data) + + def test_db_sync_contract(self): + self.command_contract = True + self.useFixture(mockpatch.PatchObject( + CONF, 'command', self.FakeConfCommand(self))) + cli.DbSync.main() + self._assert_correct_call(migration_helpers.contract_schema)