Add support for rolling upgrades to keystone-manage

Add all the commands to keystone-manage as well as stubs for
the logic in the migration helpers, for the expand, migrate
and contract cycles of a rolling upgrade.

Follow-on patchs will add the logic to the migration helpers.

Partially Implements blueprint manage-migration
Change-Id: I9f138fe0bcbf5ffbb98e6fcebd7d897329a301b7

keystone/cmd/cli.py

@@ -407,14 +407,42 @@ class DbSync(BaseApp):
                                   'now part of the main repository, '
                                   'specifying db_sync without this option '
                                   'will cause all extensions to be migrated.'))
-
+        group = parser.add_mutually_exclusive_group()
+        group.add_argument('--expand', default=False, action='store_true',
+                           help=('Expand the database schema in preparation '
+                                 'for data migration and starting the first '
+                                 'keystone node upgraded to the new release.'))
+        group.add_argument('--migrate', default=False,
+                           action='store_true',
+                           help=('Copy all data that needs to be migrated '
+                                 'within the database ahead of starting the '
+                                 'first keystone node upgraded to the new '
+                                 'release. This command should be run '
+                                 'after the --expand command. Once the '
+                                 '--migrate command has completed, you can '
+                                 'upgrade all your keystone nodes to the new '
+                                 'release and restart them.'))
+
+        group.add_argument('--contract', default=False, action='store_true',
+                           help=('Remove any database tables and columns '
+                                 'that are no longer required. This command '
+                                 'should be run after all keystone nodes are '
+                                 'running the new release.'))
         return parser
 
     @staticmethod
     def main():
         assert_not_extension(CONF.command.extension)
-        version = CONF.command.version
-        migration_helpers.sync_database_to_version(version)
+
+        if CONF.command.expand:
+            migration_helpers.expand_schema()
+        elif CONF.command.migrate:
+            migration_helpers.migrate_data()
+        elif CONF.command.contract:
+            migration_helpers.contract_schema()
+        else:
+            migration_helpers.offline_sync_database_to_version(
+                CONF.command.version)
 
 
 class DbVersion(BaseApp):

keystone/common/sql/migration_helpers.py

@@ -160,9 +160,25 @@ def _assert_not_schema_downgrade(version=None):
             pass
 
 
-def sync_database_to_version(version=None):
+def offline_sync_database_to_version(version=None):
+    """Perform and off-line sync of the database.
+
+    Migrate the database up to the latest version, doing the equivalent of
+    the cycle of --expand, --migrate and --contract, for when an offline
+    upgrade is being performed.
+
+    If a version is specified then only migrate the database up to that
+    version. Downgrading is not supported. If version is specified, then only
+    the main database migration is carried out - and the data migration and
+    contract phases will NOT be run.
+
+    """
     _sync_common_repo(version)
 
+    if not version:
+        migrate_data()
+        contract_schema()
+
 
 def get_db_version():
     with sql.session_for_write() as session:
@@ -173,3 +189,39 @@ def get_db_version():
 
 def print_db_version():
     print(get_db_version())
+
+
+def expand_schema():
+    """Expand the database schema ahead of data migration.
+
+    This is run manually by the keystone-manage command before the first
+    keystone node is migrated to the latest release.
+
+    """
+    # TODO(henry-nash): Add implementation here.
+    pass
+
+
+def migrate_data():
+    """Migrate data to match the new schema.
+
+    This is run manually by the keystone-manage command once the keystone
+    schema has been expanded for the new release.
+
+    """
+    # TODO(henry-nash): Add implementation here.
+    pass
+
+
+def contract_schema():
+    """Contract the database.
+
+    This is run manually by the keystone-manage command once the keystone
+    nodes have been upgraded to the latest release and will remove any old
+    tables/columns that are no longer required. In addition, if any data
+    could have been left inconsistent while running with a mix of releases,
+    then this should be fixed up here.
+
+    """
+    # TODO(henry-nash): Add implementation here.
+    pass

keystone/tests/unit/test_cli.py

@@ -25,6 +25,7 @@ from testtools import matchers
 
 from keystone.cmd import cli
 from keystone.common import dependency
+from keystone.common.sql import migration_helpers
 import keystone.conf
 from keystone.i18n import _
 from keystone.tests import unit
@@ -534,3 +535,64 @@ class TestDomainConfigFinder(unit.BaseTestCase):
         self.assertThat(
             self.logging.output,
             matchers.Contains(expected_msg_template % 'keystone.conf'))
+
+
+class CliDBSyncTestCase(unit.BaseTestCase):
+
+    class FakeConfCommand(object):
+        def __init__(self, parent):
+            self.extension = False
+            self.expand = parent.command_expand
+            self.migrate = parent.command_migrate
+            self.contract = parent.command_contract
+            self.version = None
+
+    def setUp(self):
+        super(CliDBSyncTestCase, self).setUp()
+        self.config_fixture = self.useFixture(config_fixture.Config(CONF))
+        self.config_fixture.register_cli_opt(cli.command_opt)
+        migration_helpers.offline_sync_database_to_version = mock.Mock()
+        migration_helpers.expand_schema = mock.Mock()
+        migration_helpers.migrate_data = mock.Mock()
+        migration_helpers.contract_schema = mock.Mock()
+        self.command_expand = False
+        self.command_migrate = False
+        self.command_contract = False
+
+    def _assert_correct_call(self, mocked_function):
+        for func in [migration_helpers.offline_sync_database_to_version,
+                     migration_helpers.expand_schema,
+                     migration_helpers.migrate_data,
+                     migration_helpers.contract_schema]:
+            if func == mocked_function:
+                self.assertTrue(func.called)
+            else:
+                self.assertFalse(func.called)
+
+    def test_db_sync(self):
+        self.useFixture(mockpatch.PatchObject(
+            CONF, 'command', self.FakeConfCommand(self)))
+        cli.DbSync.main()
+        self._assert_correct_call(
+            migration_helpers.offline_sync_database_to_version)
+
+    def test_db_sync_expand(self):
+        self.command_expand = True
+        self.useFixture(mockpatch.PatchObject(
+            CONF, 'command', self.FakeConfCommand(self)))
+        cli.DbSync.main()
+        self._assert_correct_call(migration_helpers.expand_schema)
+
+    def test_db_sync_migrate(self):
+        self.command_migrate = True
+        self.useFixture(mockpatch.PatchObject(
+            CONF, 'command', self.FakeConfCommand(self)))
+        cli.DbSync.main()
+        self._assert_correct_call(migration_helpers.migrate_data)
+
+    def test_db_sync_contract(self):
+        self.command_contract = True
+        self.useFixture(mockpatch.PatchObject(
+            CONF, 'command', self.FakeConfCommand(self)))
+        cli.DbSync.main()
+        self._assert_correct_call(migration_helpers.contract_schema)