Add expand, data migration and contract logic to keystone-manage

3 new migration repos are added, one for each of the new phases.
The existing "migrate_repo" is now frozen (except for backports).
The sql_banned operations tests are now applied both to the
frozen legacy repo and the expand repo.

This patch contains a null first migration in each repo (some
of our support methods don't handle empty repos) - follow on
patches will add actual migration scripts to these repos.

Implements: blueprint manage-migration
Change-Id: Ie68b463b7a3acbf39486d75026b80bf5dcbc5288

keystone/common/sql/contract_repo/README

@@ -0,0 +1,4 @@
+This is a database migration repository.
+
+More information at
+https://git.openstack.org/cgit/openstack/sqlalchemy-migrate

keystone/common/sql/contract_repo/manage.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+from migrate.versioning.shell import main
+
+if __name__ == '__main__':
+    main(debug='False')

keystone/common/sql/contract_repo/migrate.cfg

@@ -0,0 +1,25 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=keystone_contract
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfully during a commit, or the
+# entire commit will fail. List the databases your application will actually
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]
+
+# When creating new change scripts, Migrate will stamp the new script with
+# a version number. By default this is latest_version + 1. You can set this
+# to 'true' to tell Migrate to use the UTC timestamp instead.
+use_timestamp_numbering=False

keystone/common/sql/contract_repo/versions/001_contract_initial_null_migration.py

@@ -0,0 +1,18 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# A null initial migration to open this repo. Do not re-use replace this with
+# a real migration, add additional ones in subsequent version scripts.
+
+
+def upgrade(migrate_engine):
+    pass

keystone/common/sql/data_migration_repo/README

@@ -0,0 +1,4 @@
+This is a database migration repository.
+
+More information at
+https://git.openstack.org/cgit/openstack/sqlalchemy-migrate

keystone/common/sql/data_migration_repo/manage.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+from migrate.versioning.shell import main
+
+if __name__ == '__main__':
+    main(debug='False')

keystone/common/sql/data_migration_repo/migrate.cfg

@@ -0,0 +1,25 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=keystone_data_migrate
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfully during a commit, or the
+# entire commit will fail. List the databases your application will actually
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]
+
+# When creating new change scripts, Migrate will stamp the new script with
+# a version number. By default this is latest_version + 1. You can set this
+# to 'true' to tell Migrate to use the UTC timestamp instead.
+use_timestamp_numbering=False

keystone/common/sql/data_migration_repo/versions/001_data_initial_null_migration.py

@@ -0,0 +1,18 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# A null initial migration to open this repo. Do not re-use replace this with
+# a real migration, add additional ones in subsequent version scripts.
+
+
+def upgrade(migrate_engine):
+    pass

keystone/common/sql/expand_repo/README

@@ -0,0 +1,4 @@
+This is a database migration repository.
+
+More information at
+https://git.openstack.org/cgit/openstack/sqlalchemy-migrate

keystone/common/sql/expand_repo/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2012 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from keystone.common.sql.core import *  # noqa

keystone/common/sql/expand_repo/manage.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+from migrate.versioning.shell import main
+
+if __name__ == '__main__':
+    main(debug='False')

keystone/common/sql/expand_repo/migrate.cfg

@@ -0,0 +1,25 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=keystone_expand
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfully during a commit, or the
+# entire commit will fail. List the databases your application will actually
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]
+
+# When creating new change scripts, Migrate will stamp the new script with
+# a version number. By default this is latest_version + 1. You can set this
+# to 'true' to tell Migrate to use the UTC timestamp instead.
+use_timestamp_numbering=False

keystone/common/sql/expand_repo/versions/001_expand_initial_null_migration.py

@@ -0,0 +1,18 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# A null initial migration to open this repo. Do not re-use replace this with
+# a real migration, add additional ones in subsequent version scripts.
+
+
+def upgrade(migrate_engine):
+    pass

keystone/common/sql/expand_repo/versions/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2012 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from keystone.common.sql.core import *  # noqa

keystone/common/sql/migration_helpers.py

@@ -125,6 +125,23 @@ def _sync_common_repo(version):
                           init_version=init_version, sanity_check=False)
 
 
+def _sync_repo(repo_name):
+    abs_path = find_migrate_repo(repo_name=repo_name)
+    with sql.session_for_write() as session:
+        engine = session.get_bind()
+        # Register the repo with the version control API
+        # If it already knows about the repo, it will throw
+        # an exception that we can safely ignore
+        try:
+            migration.db_version_control(engine, abs_path)
+        except (migration.exception.DbMigrationError,
+                exceptions.DatabaseAlreadyControlledError):  # nosec
+            pass
+        init_version = get_init_version(abs_path=abs_path)
+        migration.db_sync(engine, abs_path,
+                          init_version=init_version, sanity_check=False)
+
+
 def get_init_version(abs_path=None):
     """Get the initial version of a migrate repository.
 
@@ -169,13 +186,14 @@ def offline_sync_database_to_version(version=None):
 
     If a version is specified then only migrate the database up to that
     version. Downgrading is not supported. If version is specified, then only
-    the main database migration is carried out - and the data migration and
+    the main database migration is carried out - and the expand, migration and
     contract phases will NOT be run.
 
     """
+    if version:
         _sync_common_repo(version)
-
+    else:
-    if not version:
+        expand_schema()
         migrate_data()
         contract_schema()
 
@@ -198,8 +216,10 @@ def expand_schema():
     keystone node is migrated to the latest release.
 
     """
-    # TODO(henry-nash): Add implementation here.
+    # Make sure all the legacy migrations are run before we run any new
-    pass
+    # expand migrations.
+    _sync_common_repo(version=None)
+    _sync_repo(repo_name='expand_repo')
 
 
 def migrate_data():
@@ -209,8 +229,7 @@ def migrate_data():
     schema has been expanded for the new release.
 
     """
-    # TODO(henry-nash): Add implementation here.
+    _sync_repo(repo_name='data_migration_repo')
-    pass
 
 
 def contract_schema():
@@ -223,5 +242,4 @@ def contract_schema():
     then this should be fixed up here.
 
     """
-    # TODO(henry-nash): Add implementation here.
+    _sync_repo(repo_name='contract_repo')
-    pass

keystone/tests/unit/test_sql_banned_operations.py

@@ -22,6 +22,7 @@ from oslo_db.sqlalchemy import test_migrations
 import sqlalchemy
 import testtools
 
+from keystone.common.sql import expand_repo
 from keystone.common.sql import migrate_repo
 from keystone.common.sql import migration_helpers
 
@@ -173,3 +174,51 @@ class TestKeystoneMigrationsPostgreSQL(
 class TestKeystoneMigrationsSQLite(
         KeystoneMigrationsCheckers, test_base.DbTestCase):
     pass
+
+
+class TestKeystoneExpandSchemaMigrationsMySQL(
+        KeystoneMigrationsCheckers, test_base.MySQLOpportunisticTestCase):
+
+    @property
+    def INIT_VERSION(self):
+        return migration_helpers.get_init_version(
+            abs_path=os.path.abspath(os.path.dirname(expand_repo.__file__)))
+
+    @property
+    def REPOSITORY(self):
+        migrate_file = expand_repo.__file__
+        return repository.Repository(
+            os.path.abspath(os.path.dirname(migrate_file))
+        )
+
+
+class TestKeystoneExpandSchemaMigrationsPostgreSQL(
+        KeystoneMigrationsCheckers, test_base.PostgreSQLOpportunisticTestCase):
+
+    @property
+    def INIT_VERSION(self):
+        return migration_helpers.get_init_version(
+            abs_path=os.path.abspath(os.path.dirname(expand_repo.__file__)))
+
+    @property
+    def REPOSITORY(self):
+        migrate_file = expand_repo.__file__
+        return repository.Repository(
+            os.path.abspath(os.path.dirname(migrate_file))
+        )
+
+
+class TestKeystoneExpandSchemaMigrationsSQLite(
+        KeystoneMigrationsCheckers, test_base.DbTestCase):
+
+    @property
+    def INIT_VERSION(self):
+        return migration_helpers.get_init_version(
+            abs_path=os.path.abspath(os.path.dirname(expand_repo.__file__)))
+
+    @property
+    def REPOSITORY(self):
+        migrate_file = expand_repo.__file__
+        return repository.Repository(
+            os.path.abspath(os.path.dirname(migrate_file))
+        )

keystone/tests/unit/test_sql_upgrade.py

@@ -119,6 +119,11 @@ INITIAL_TABLE_STRUCTURE = {
     ],
 }
 
+LEGACY_REPO = 'migrate_repo'
+EXPAND_REPO = 'expand_repo'
+DATA_MIGRATION_REPO = 'data_migration_repo'
+CONTRACT_REPO = 'contract_repo'
+
 
 # Test migration_helpers.get_init_version separately to ensure it works before
 # using in the SqlUpgrade tests.
@@ -142,7 +147,7 @@ class MigrationHelpersGetInitVersionTests(unit.TestCase):
         # first invocation of repo. Cannot match the full path because it is
         # based on where the test is run.
         param = repo.call_args_list[0][0][0]
-        self.assertTrue(param.endswith('/sql/migrate_repo'))
+        self.assertTrue(param.endswith('/sql/' + LEGACY_REPO))
 
     @mock.patch.object(repository, 'Repository')
     def test_get_init_version_with_path_initial_version_0(self, repo):
@@ -155,7 +160,7 @@ class MigrationHelpersGetInitVersionTests(unit.TestCase):
         # os.path.isdir() is called by `find_migrate_repo()`. Mock it to avoid
         # an exception.
         with mock.patch('os.path.isdir', return_value=True):
-            path = '/keystone/migrate_repo/'
+            path = '/keystone/' + LEGACY_REPO + '/'
 
             # since 0 is the smallest version expect None
             version = migration_helpers.get_init_version(abs_path=path)
@@ -173,7 +178,7 @@ class MigrationHelpersGetInitVersionTests(unit.TestCase):
         # os.path.isdir() is called by `find_migrate_repo()`. Mock it to avoid
         # an exception.
         with mock.patch('os.path.isdir', return_value=True):
-            path = '/keystone/migrate_repo/'
+            path = '/keystone/' + LEGACY_REPO + '/'
 
             version = migration_helpers.get_init_version(abs_path=path)
             self.assertEqual(initial_version, version)
@@ -191,6 +196,18 @@ class SqlMigrateBase(test_base.DbTestCase):
     def repo_package(self):
         return sql
 
+    def initialize_repo(self, repo_name=LEGACY_REPO):
+        self.repo_path = migration_helpers.find_migrate_repo(
+            package=self.repo_package(),
+            repo_name=repo_name)
+        self._initial_db_version = (
+            migration_helpers.get_init_version(abs_path=self.repo_path))
+        self.schema_ = versioning_api.ControlledSchema.create(
+            self.engine,
+            self.repo_path,
+            self._initial_db_version)
+        self.max_version = self.schema_.repository.version().version
+
     def setUp(self):
         super(SqlMigrateBase, self).setUp()
 
@@ -205,15 +222,7 @@ class SqlMigrateBase(test_base.DbTestCase):
         self.addCleanup(sql.cleanup)
 
         self.initialize_sql()
-        self.repo_path = migration_helpers.find_migrate_repo(
+        self.initialize_repo()
-            self.repo_package())
-        self.schema_ = versioning_api.ControlledSchema.create(
-            self.engine,
-            self.repo_path,
-            self._initial_db_version)
-
-        # auto-detect the highest available schema version in the migrate_repo
-        self.max_version = self.schema_.repository.version().version
 
     def select_table(self, name):
         table = sqlalchemy.Table(name,
@@ -285,8 +294,18 @@ class SqlMigrateBase(test_base.DbTestCase):
         self.assertItemsEqual(expected_cols, actual_cols,
                               '%s table' % table_name)
 
+    def insert_dict(self, session, table_name, d, table=None):
+        """Naively inserts key-value pairs into a table, given a dictionary."""
+        if table is None:
+            this_table = sqlalchemy.Table(table_name, self.metadata,
+                                          autoload=True)
+        else:
+            this_table = table
+        insert = this_table.insert().values(**d)
+        session.execute(insert)
 
-class SqlUpgradeTests(SqlMigrateBase):
+
+class SqlLegacyRepoUpgradeTests(SqlMigrateBase):
     _initial_db_version = migration_helpers.get_init_version()
 
     def test_blank_db_to_start(self):
@@ -309,16 +328,6 @@ class SqlUpgradeTests(SqlMigrateBase):
         for table in INITIAL_TABLE_STRUCTURE:
             self.assertTableColumns(table, INITIAL_TABLE_STRUCTURE[table])
 
-    def insert_dict(self, session, table_name, d, table=None):
-        """Naively inserts key-value pairs into a table, given a dictionary."""
-        if table is None:
-            this_table = sqlalchemy.Table(table_name, self.metadata,
-                                          autoload=True)
-        else:
-            this_table = table
-        insert = this_table.insert().values(**d)
-        session.execute(insert)
-
     def test_kilo_squash(self):
         self.upgrade(67)
 
@@ -1480,11 +1489,110 @@ class SqlUpgradeTests(SqlMigrateBase):
                                  'failed_auth_at'])
 
 
-class MySQLOpportunisticUpgradeTestCase(SqlUpgradeTests):
+class MySQLOpportunisticUpgradeTestCase(SqlLegacyRepoUpgradeTests):
     FIXTURE = test_base.MySQLOpportunisticFixture
 
 
-class PostgreSQLOpportunisticUpgradeTestCase(SqlUpgradeTests):
+class PostgreSQLOpportunisticUpgradeTestCase(SqlLegacyRepoUpgradeTests):
+    FIXTURE = test_base.PostgreSQLOpportunisticFixture
+
+
+class SqlExpandSchemaUpgradeTests(SqlMigrateBase):
+
+    def setUp(self):
+        # Make sure the main repo is fully upgraded for this release since the
+        # expand phase is only run after such an upgrade
+        super(SqlExpandSchemaUpgradeTests, self).setUp()
+        self.upgrade(self.max_version)
+
+        self.initialize_repo(repo_name=EXPAND_REPO)
+
+    def test_start_version_db_init_version(self):
+        with sql.session_for_write() as session:
+            version = migration.db_version(session.get_bind(), self.repo_path,
+                                           self._initial_db_version)
+        self.assertEqual(
+            self._initial_db_version,
+            version,
+            'DB is not at version %s' % self._initial_db_version)
+
+
+class MySQLOpportunisticExpandSchemaUpgradeTestCase(
+        SqlExpandSchemaUpgradeTests):
+    FIXTURE = test_base.MySQLOpportunisticFixture
+
+
+class PostgreSQLOpportunisticExpandSchemaUpgradeTestCase(
+        SqlExpandSchemaUpgradeTests):
+    FIXTURE = test_base.PostgreSQLOpportunisticFixture
+
+
+class SqlDataMigrationUpgradeTests(SqlMigrateBase):
+
+    def setUp(self):
+        # Make sure the legacy and expand repos are fully upgraded, since the
+        # data migration phase is only run after these are upgraded
+        super(SqlDataMigrationUpgradeTests, self).setUp()
+        self.upgrade(self.max_version)
+        # Make sure the expand repo is also upgraded
+        self.initialize_repo(repo_name=EXPAND_REPO)
+        self.upgrade(self.max_version)
+
+        self.initialize_repo(repo_name=DATA_MIGRATION_REPO)
+
+    def test_start_version_db_init_version(self):
+        with sql.session_for_write() as session:
+            version = migration.db_version(session.get_bind(), self.repo_path,
+                                           self._initial_db_version)
+        self.assertEqual(
+            self._initial_db_version,
+            version,
+            'DB is not at version %s' % self._initial_db_version)
+
+
+class MySQLOpportunisticDataMigrationUpgradeTestCase(
+        SqlDataMigrationUpgradeTests):
+    FIXTURE = test_base.MySQLOpportunisticFixture
+
+
+class PostgreSQLOpportunisticDataMigrationUpgradeTestCase(
+        SqlDataMigrationUpgradeTests):
+    FIXTURE = test_base.PostgreSQLOpportunisticFixture
+
+
+class SqlContractSchemaUpgradeTests(SqlMigrateBase):
+
+    def setUp(self):
+        # Make sure the legacy, expand and data migration repos are fully
+        # upgraded, since the contract phase is only run after these are
+        # upgraded.
+        super(SqlContractSchemaUpgradeTests, self).setUp()
+        self.upgrade(self.max_version)
+
+        self.initialize_repo(repo_name=EXPAND_REPO)
+        self.upgrade(self.max_version)
+        self.initialize_repo(repo_name=DATA_MIGRATION_REPO)
+        self.upgrade(self.max_version)
+
+        self.initialize_repo(repo_name=CONTRACT_REPO)
+
+    def test_start_version_db_init_version(self):
+        with sql.session_for_write() as session:
+            version = migration.db_version(session.get_bind(), self.repo_path,
+                                           self._initial_db_version)
+        self.assertEqual(
+            self._initial_db_version,
+            version,
+            'DB is not at version %s' % self._initial_db_version)
+
+
+class MySQLOpportunisticContractSchemaUpgradeTestCase(
+        SqlContractSchemaUpgradeTests):
+    FIXTURE = test_base.MySQLOpportunisticFixture
+
+
+class PostgreSQLOpportunisticContractSchemaUpgradeTestCase(
+        SqlContractSchemaUpgradeTests):
     FIXTURE = test_base.PostgreSQLOpportunisticFixture
 
 

releasenotes/notes/bp-manage-migration-c398963a943a89fe.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - >
+    [`blueprint manage-migration <https://blueprints.launchpad.net/keystone/+spec/manage-migration>`_]
+    Upgrading keystone to a new version can now be undertaken as a rolling
+    upgrade using the `--expand`, `--migrate` and `--contract` options of the
+    `keystone-manage db_sync` command.